import pandas as pd

# 讀取'test_data.xlsx'大文件

# 分塊讀取以減少內(nèi)存占用
chunk_size = 10000
chunks = pd.read_excel('test_data.xlsx', chunksize=chunk_size)

# 對(duì)每個(gè)數(shù)據(jù)塊進(jìn)行處理
processed_data = []
for i, chunk in enumerate(chunks):
    這里代碼是對(duì)每個(gè)數(shù)據(jù)塊進(jìn)行處理......

from openpyxl import load_workbook

# 使用只讀模式讀取大型Excel文件
read_wb = load_workbook('large_data.xlsx', read_only=True)
read_ws = read_wb.active

# 以流式方式逐行讀取、處理和寫入
row_count = 0
for row in read_ws.iter_rows(min_row=2): 
    這里代碼是對(duì)每一行進(jìn)行處理......

import xlsxwriter

# 創(chuàng)建一個(gè)文件名，用于寫入數(shù)據(jù)
file_name = 'large_data_constant_memory.xlsx'

# 創(chuàng)建一個(gè)工作簿對(duì)象，并啟用常量?jī)?nèi)存模式
workbook = xlsxwriter.Workbook(file_name, {'constant_memory': True})
worksheet = workbook.add_worksheet()
這里可以寫入大量數(shù)據(jù)，不會(huì)占用內(nèi)存......

with open('large_file.txt', 'r') as file:
for line in file:
process(line) # Replace with your processing function

def read_file_in_chunks(file_path, chunk_size=1024):
with open(file_path, 'r') as file:
while True:
chunk = file.read(chunk_size)
if not chunk:
break
process(chunk) # Replace with your processing function

with open('large_file.txt', 'rb', buffering=10 * 1024 * 1024) as file: # 10 MB buffer
for line in file:
process(line)

from mmap import mmap

with open('large_file.txt', 'r') as file:
with mmap(file.fileno(), length=0, access=mmap.ACCESS_READ) as mm:
for line in mm:
process(line.decode('utf-8'))

def generate_lines(file_path):
with open(file_path, 'r') as file:
for line in file:
yield line

for line in generate_lines('large_file.txt'):
process(line)

def read_batches(file_path, batch_size=5):
with open(file_path, 'r') as file:
batch = []
for line in file:
batch.append(line.strip())
if len(batch) == batch_size:
yield batch
batch = []
if batch:
yield batch

# Example usage:
for batch in read_batches('cars.txt'):
process_batch(batch) # Replace with your processing logic

import requests

def stream_data(url):
with requests.get(url, stream=True) as response:
for line in response.iter_lines():
process(line)

import dask.dataframe as dd

df = dd.read_csv('large_dataset.csv')
result = df[df['column'] > 100].compute()

from pyspark.sql import SparkSession

spark = SparkSession.builder.appName("LargeFileProcessing").getOrCreate()
df = spark.read.csv('large_dataset.csv')
df.filter(df['column'] > 100).show()