# 傳統(tǒng)DOM解析 - 內(nèi)存密集型
import xml.dom.minidom as minidom

# 對(duì)于大文件，這將消耗大量內(nèi)存
dom = minidom.parse('large_file.xml')  # 不推薦用于大文件

import xml.etree.ElementTree as ET

# 基本迭代解析
context = ET.iterparse('large_data.xml', events=('start', 'end'))

for event, elem in context:
    if event == 'start':
        print(f"開始元素: {elem.tag}")
    elif event == 'end':
        print(f"結(jié)束元素: {elem.tag}")
        # 處理完成后清除元素以釋放內(nèi)存
        elem.clear()

def parse_and_remove(filename, path):
    """增量解析并移除已處理元素"""
    path_parts = path.split('/')
    doc = ET.iterparse(filename, ('start', 'end'))
    
    # 跳過根元素
    next(doc)
    
    tag_stack = []
    elem_stack = []
    
    for event, elem in doc:
        if event == 'start':
            tag_stack.append(elem.tag)
            elem_stack.append(elem)
        elif event == 'end':
            if tag_stack == path_parts:
                yield elem
                # 關(guān)鍵步驟：從父元素中移除已處理的元素
                elem_stack[-2].remove(elem)
            try:
                tag_stack.pop()
                elem_stack.pop()
            except IndexError:
                pass

# 使用示例
for elem in parse_and_remove('huge_data.xml', 'row/row'):
    # 處理每個(gè)row元素
    zip_code = elem.findtext('zip')
    process_data(zip_code)  # 自定義處理函數(shù)

context = ET.iterparse('large_file.xml', events=('end',))

for event, elem in context:
    if event == 'end' and elem.tag == 'record':
        # 處理記錄
        process_record(elem)
        
        # 關(guān)鍵：清除已處理的元素
        elem.clear()
        
        # 可選：清除父元素中的空引用
        if elem.getparent() is not None:
            del elem.getparent()[elem.index:]

from lxml import etree

# lxml的迭代解析，性能更好
context = etree.iterparse('very_large_file.xml', 
                         events=('end',), 
                         tag='record')

for event, elem in context:
    try:
        # 處理元素
        data = extract_data(elem)
        yield data
    finally:
        # 清除元素并釋放內(nèi)存
        elem.clear()
        while elem.getprevious() is not None:
            del elem.getparent()[0]

# 處理帶命名空間的XML
def parse_with_namespace(filename, element_name):
    # 自動(dòng)檢測(cè)命名空間
    for _, elem in ET.iterparse(filename, events=('end',)):
        if '}' in elem.tag:
            namespace, local_name = elem.tag.split('}', 1)
            if local_name == element_name:
                yield elem
                elem.clear()
        else:
            if elem.tag == element_name:
                yield elem
                elem.clear()

# 使用顯式命名空間
namespaces = {'ns': 'http://example.com/namespace'}
context = ET.iterparse('data.xml', events=('end',))

for event, elem in context:
    if elem.tag == '{http://example.com/namespace}record':
        process_element(elem)
        elem.clear()

def parse_deep_nested_xml(filename, target_tag):
    # 使用棧跟蹤解析深度
    depth = 0
    target_depth = None
    
    for event, elem in ET.iterparse(filename, events=('start', 'end')):
        if event == 'start':
            depth += 1
            if elem.tag == target_tag and target_depth is None:
                target_depth = depth
        elif event == 'end':
            if depth == target_depth:
                # 處理目標(biāo)元素
                yield elem
                # 清除并移除元素
                elem.clear()
                if elem.getparent() is not None:
                    elem.getparent().remove(elem)
            depth -= 1

# 只需要元素內(nèi)容時(shí)，只需監(jiān)聽end事件
context = ET.iterparse('data.xml', events=('end',))

# 需要屬性或結(jié)構(gòu)信息時(shí)，需要監(jiān)聽start和end事件
context = ET.iterparse('data.xml', events=('start', 'end'))

# 處理命名空間聲明
context = ET.iterparse('data.xml', events=('start-ns', 'end-ns', 'end'))

def batch_process_xml(filename, batch_size=1000):
    batch = []
    context = ET.iterparse(filename, events=('end',), tag='item')
    
    for event, elem in context:
        # 提取數(shù)據(jù)
        data = extract_item_data(elem)
        batch.append(data)
        
        # 清除元素
        elem.clear()
        
        # 批量處理
        if len(batch) >= batch_size:
            process_batch(batch)
            batch = []
    
    # 處理剩余數(shù)據(jù)
    if batch:
        process_batch(batch)

from multiprocessing import Pool
import glob

def process_single_xml(filename):
    """處理單個(gè)XML文件"""
    data = []
    context = ET.iterparse(filename, events=('end',), tag='record')
    
    for event, elem in context:
        data.append(extract_data(elem))
        elem.clear()
    
    return data

def process_xml_files_parallel(pattern, processes=4):
    """并行處理多個(gè)XML文件"""
    files = glob.glob(pattern)
    
    with Pool(processes=processes) as pool:
        results = pool.map(process_single_xml, files)
    
    return results

from collections import Counter
import xml.etree.ElementTree as ET

def count_potholes_by_zip(filename):
    """統(tǒng)計(jì)每個(gè)郵政編碼的坑洞數(shù)量"""
    potholes_by_zip = Counter()
    
    # 增量解析，內(nèi)存友好
    for event, elem in ET.iterparse(filename, events=('end',)):
        if elem.tag == 'row':
            zip_code = elem.findtext('zip')
            if zip_code:
                potholes_by_zip[zip_code] += 1
            
            # 關(guān)鍵：及時(shí)清除已處理元素
            elem.clear()
    
    return potholes_by_zip

# 使用示例
pothole_counts = count_potholes_by_zip('chicago_potholes.xml')
for zip_code, count in pothole_counts.most_common(10):
    print(f"ZIP: {zip_code}, 坑洞數(shù)量: {count}")

import json

def xml_to_jsonl(xml_file, jsonl_file, record_tag='record'):
    """將XML轉(zhuǎn)換為JSON Lines格式"""
    with open(jsonl_file, 'w', encoding='utf-8') as outf:
        context = ET.iterparse(xml_file, events=('end',), tag=record_tag)
        
        for event, elem in context:
            # 將元素轉(zhuǎn)換為字典
            record = element_to_dict(elem)
            
            # 寫入JSONL文件
            outf.write(json.dumps(record, ensure_ascii=False) + '\n')
            
            # 清除元素
            elem.clear()

def element_to_dict(elem):
    """將XML元素轉(zhuǎn)換為字典"""
    result = {}
    
    # 處理屬性
    if elem.attrib:
        result['@attributes'] = elem.attrib
    
    # 處理子元素
    for child in elem:
        child_data = element_to_dict(child)
        
        if child.tag in result:
            # 轉(zhuǎn)換為列表處理多個(gè)相同標(biāo)簽
            if not isinstance(result[child.tag], list):
                result[child.tag] = [result[child.tag]]
            result[child.tag].append(child_data)
        else:
            result[child.tag] = child_data
    
    # 處理文本內(nèi)容
    if elem.text and elem.text.strip():
        if result:  # 既有屬性/子元素又有文本
            result['#text'] = elem.text
        else:
            result = elem.text
    
    return result

def robust_iterparse(filename, events=('end',), tag='record'):
    """健壯的迭代解析，處理損壞數(shù)據(jù)"""
    try:
        context = ET.iterparse(filename, events=events, tag=tag)
        
        for event, elem in context:
            try:
                yield elem
            except Exception as e:
                print(f"處理元素時(shí)出錯(cuò): {e}")
                # 繼續(xù)處理下一個(gè)元素
                continue
            finally:
                elem.clear()
    except ET.ParseError as e:
        print(f"XML解析錯(cuò)誤: {e}")
        # 可以在這里實(shí)現(xiàn)恢復(fù)邏輯
    except Exception as e:
        print(f"未知錯(cuò)誤: {e}")

def resume_parsing(filename, last_processed_id=None):
    """從斷點(diǎn)處恢復(fù)解析"""
    context = ET.iterparse(filename, events=('end',), tag='record')
    
    resume = (last_processed_id is None)
    
    for event, elem in context:
        if not resume:
            current_id = elem.findtext('id')
            if current_id == last_processed_id:
                resume = True
            elem.clear()
            continue
        
        try:
            # 處理元素
            process_record(elem)
            last_id = elem.findtext('id')
            
            # 定期保存進(jìn)度
            save_progress(last_id)
            
        finally:
            elem.clear()