python批量處理PDF文檔輸出自定義關(guān)鍵詞的出現(xiàn)次數(shù)

更新時(shí)間：2023年04月11日 11:54:12 作者：Ryo_Yuki

這篇文章主要介紹了python批量處理PDF文檔，輸出自定義關(guān)鍵詞的出現(xiàn)次數(shù)，文中有詳細(xì)的代碼示例，需要的朋友可以參考閱讀

函數(shù)模塊介紹

具體的代碼可見全部代碼部分，這部分只介紹思路和相應(yīng)的函數(shù)模塊

對文件進(jìn)行批量重命名

因?yàn)槲募侵形模覠o關(guān)于最后的結(jié)果，所以批量命名為數(shù)字
注意如果不是第一次運(yùn)行，即已經(jīng)命名完成，就在主函數(shù)內(nèi)把這個(gè)函數(shù)注釋掉就好了

def rename():
    path='dealPdf'
    filelist=os.listdir(path)
    for i,files in enumerate(filelist):
        Olddir=os.path.join(path,files)
        if os.path.isdir(Olddir):
            continue
        Newdir=os.path.join(path,str(i+1)+'.pdf')
        os.rename(Olddir,Newdir)

將PDF轉(zhuǎn)化為txt

PDF是無法直接進(jìn)行文本分析的，所以需要將文字轉(zhuǎn)成txt文件（PDF中圖內(nèi)的文字無法提取）

#將pdf文件轉(zhuǎn)化成txt文件
def pdf_to_txt(dealPdf,index):
    # 不顯示warning
    logging.propagate = False
    logging.getLogger().setLevel(logging.ERROR)
    pdf_filename = dealPdf
    device = PDFPageAggregator(PDFResourceManager(), laparams=LAParams())
    interpreter = PDFPageInterpreter(PDFResourceManager(), device)    
    parser = PDFParser(open(pdf_filename, 'rb'))
    doc = PDFDocument(parser)
    
    
    txt_filename='dealTxt\\'+str(index)+'.txt'
        
    # 檢測文檔是否提供txt轉(zhuǎn)換，不提供就忽略
    if not doc.is_extractable:
        raise PDFTextExtractionNotAllowed
    else:
        with open(txt_filename, 'w', encoding="utf-8") as fw:
            #print("num page:{}".format(len(list(doc.get_pages()))))
            for i,page in enumerate(PDFPage.create_pages(doc)):
                interpreter.process_page(page)
                # 接受該頁面的LTPage對象
                layout = device.get_result()
                # 這里layout是一個(gè)LTPage對象 里面存放著 這個(gè)page解析出的各種對象
                # 一般包括LTTextBox, LTFigure, LTImage, LTTextBoxHorizontal 等等
                # 想要獲取文本就獲得對象的text屬性，
                for x in layout:
                    if isinstance(x, LTTextBoxHorizontal):
                        results = x.get_text()
                        fw.write(results)

刪除txt中的換行符

因?yàn)镻DF導(dǎo)出的txt會(huì)用換行符換行，為了避免詞語因此拆開，所以刪除所有的換行符

#對txt文件的換行符進(jìn)行刪除
def delete_huanhangfu(dealTxt,index):
    outPutString=''
    outPutTxt='outPutTxt\\'+str(index)+'.txt'
    with open(dealTxt,'r',encoding="utf-8") as f:
        lines=f.readlines()
        for i in range(len(lines)):
            if lines[i].endswith('\n'):
                lines[i]=lines[i][:-1] #將字符串末尾的\n去掉
        for j in range(len(lines)):
            outPutString+=lines[j]
    with open(outPutTxt,'w',encoding="utf-8") as fw:
        fw.write(outPutString)

添加自定義詞語

此處可以根據(jù)自己的需要自定義，傳入的wordsByMyself是全局變量

分詞與詞頻統(tǒng)計(jì)

調(diào)用jieba進(jìn)行分詞，讀取通用詞表去掉停用詞（此步其實(shí)可以省略，對最終結(jié)果影響不大），將詞語和出現(xiàn)次數(shù)合成為鍵值對，輸出關(guān)鍵詞出現(xiàn)次數(shù)

#分詞并進(jìn)行詞頻統(tǒng)計(jì)
def cut_and_count(outPutTxt):
    with open(outPutTxt,encoding='utf-8') as f: 
        #step1：讀取文檔并調(diào)用jieba分詞
        text=f.read() 
        words=jieba.lcut(text)
        #step2:讀取停用詞表，去停用詞
        stopwords = {}.fromkeys([ line.rstrip() for line in open('stopwords.txt',encoding='utf-8') ])
        finalwords = []
        for word in words:
            if word not in stopwords:
                if (word != "。" and word != "，") :
                    finalwords.append(word)       
        
        
        #step3：統(tǒng)計(jì)特定關(guān)鍵詞的出現(xiàn)次數(shù)
        valuelist=[0]*len(wordsByMyself)
        counts=dict(zip(wordsByMyself,valuelist))
        for word in finalwords:
            if len(word) == 1:#單個(gè)詞不計(jì)算在內(nèi)
                continue
            else:
                counts[word]=counts.get(word,0)+1#遍歷所有詞語，每出現(xiàn)一次其對應(yīng)值加1
        for i in range(len(wordsByMyself)):
            if wordsByMyself[i] in counts:
                print(wordsByMyself[i]+':'+str(counts[wordsByMyself[i]]))
            else:
                print(wordsByMyself[i]+':0')

主函數(shù)

通過for循環(huán)進(jìn)行批量操作

if __name__ == "__main__":
    #rename()   
    for i in range(1,fileNum+1):
        pdf_to_txt('dealPdf\\'+str(i)+'.pdf',i)#將pdf文件轉(zhuǎn)化成txt文件，傳入文件路徑 
        delete_huanhangfu('dealTxt\\'+str(i)+'.txt',i)#對txt文件的換行符進(jìn)行刪除，防止詞語因換行被拆分
        word_by_myself()#添加自定義詞語
        print(f'----------result {i}----------')
        cut_and_count('outPutTxt\\'+str(i)+'.txt')#分詞并進(jìn)行詞頻統(tǒng)計(jì)，傳入文件路徑

本地文件結(jié)構(gòu)

全部代碼

import jieba
import jieba.analyse
from pdfminer.pdfparser import PDFParser
from pdfminer.pdfdocument import PDFDocument
from pdfminer.pdfinterp import PDFResourceManager, PDFPageInterpreter
from pdfminer.converter import PDFPageAggregator
from pdfminer.layout import LTTextBoxHorizontal, LAParams
from pdfminer.pdfpage import PDFPage,PDFTextExtractionNotAllowed
import logging
import os

wordsByMyself=['社會(huì)責(zé)任','義務(wù)','上市','公司'] #自定義詞語,全局變量
fileNum=16#存儲(chǔ)總共待處理的文件數(shù)量

#重命名所有文件夾下的文件，適應(yīng)處理需要
def rename():
    path='dealPdf'
    filelist=os.listdir(path)
    for i,files in enumerate(filelist):
        Olddir=os.path.join(path,files)
        if os.path.isdir(Olddir):
            continue
        Newdir=os.path.join(path,str(i+1)+'.pdf')
        os.rename(Olddir,Newdir)

#將pdf文件轉(zhuǎn)化成txt文件
def pdf_to_txt(dealPdf,index):
    # 不顯示warning
    logging.propagate = False
    logging.getLogger().setLevel(logging.ERROR)
    pdf_filename = dealPdf
    device = PDFPageAggregator(PDFResourceManager(), laparams=LAParams())
    interpreter = PDFPageInterpreter(PDFResourceManager(), device)    
    parser = PDFParser(open(pdf_filename, 'rb'))
    doc = PDFDocument(parser)
    
    
    txt_filename='dealTxt\\'+str(index)+'.txt'
        
    # 檢測文檔是否提供txt轉(zhuǎn)換，不提供就忽略
    if not doc.is_extractable:
        raise PDFTextExtractionNotAllowed
    else:
        with open(txt_filename, 'w', encoding="utf-8") as fw:
            #print("num page:{}".format(len(list(doc.get_pages()))))
            for i,page in enumerate(PDFPage.create_pages(doc)):
                interpreter.process_page(page)
                # 接受該頁面的LTPage對象
                layout = device.get_result()
                # 這里layout是一個(gè)LTPage對象 里面存放著 這個(gè)page解析出的各種對象
                # 一般包括LTTextBox, LTFigure, LTImage, LTTextBoxHorizontal 等等
                # 想要獲取文本就獲得對象的text屬性，
                for x in layout:
                    if isinstance(x, LTTextBoxHorizontal):
                        results = x.get_text()
                        fw.write(results)

#對txt文件的換行符進(jìn)行刪除
def delete_huanhangfu(dealTxt,index):
    outPutString=''
    outPutTxt='outPutTxt\\'+str(index)+'.txt'
    with open(dealTxt,'r',encoding="utf-8") as f:
        lines=f.readlines()
        for i in range(len(lines)):
            if lines[i].endswith('\n'):
                lines[i]=lines[i][:-1] #將字符串末尾的\n去掉
        for j in range(len(lines)):
            outPutString+=lines[j]
    with open(outPutTxt,'w',encoding="utf-8") as fw:
        fw.write(outPutString)
            
#添加自定義詞語    
def word_by_myself():
    for i in range(len(wordsByMyself)):
        jieba.add_word(wordsByMyself[i])

#分詞并進(jìn)行詞頻統(tǒng)計(jì)
def cut_and_count(outPutTxt):
    with open(outPutTxt,encoding='utf-8') as f: 
        #step1：讀取文檔并調(diào)用jieba分詞
        text=f.read() 
        words=jieba.lcut(text)
        #step2:讀取停用詞表，去停用詞
        stopwords = {}.fromkeys([ line.rstrip() for line in open('stopwords.txt',encoding='utf-8') ])
        finalwords = []
        for word in words:
            if word not in stopwords:
                if (word != "。" and word != "，") :
                    finalwords.append(word)       
        
        
        #step3：統(tǒng)計(jì)特定關(guān)鍵詞的出現(xiàn)次數(shù)
        valuelist=[0]*len(wordsByMyself)
        counts=dict(zip(wordsByMyself,valuelist))
        for word in finalwords:
            if len(word) == 1:#單個(gè)詞不計(jì)算在內(nèi)
                continue
            else:
                counts[word]=counts.get(word,0)+1#遍歷所有詞語，每出現(xiàn)一次其對應(yīng)值加1
        for i in range(len(wordsByMyself)):
            if wordsByMyself[i] in counts:
                print(wordsByMyself[i]+':'+str(counts[wordsByMyself[i]]))
            else:
                print(wordsByMyself[i]+':0')

#主函數(shù) 
if __name__ == "__main__":
    rename()   
    for i in range(1,fileNum+1):
        pdf_to_txt('dealPdf\\'+str(i)+'.pdf',i)#將pdf文件轉(zhuǎn)化成txt文件，傳入文件路徑 
        delete_huanhangfu('dealTxt\\'+str(i)+'.txt',i)#對txt文件的換行符進(jìn)行刪除，防止詞語因換行被拆分
        word_by_myself()#添加自定義詞語
        print(f'----------result {i}----------')
        cut_and_count('outPutTxt\\'+str(i)+'.txt')#分詞并進(jìn)行詞頻統(tǒng)計(jì)，傳入文件路徑