「Python 数据处理基础」英文文本数据预处理操作的9种常用方法

淩亂°似流年 2022-10-08 05:38 838阅读 0赞

### 文章目录 ###

*  内容介绍
 *  文本数据预处理准备

# 内容介绍 #

在日常的数据分析工作中，不管在处理中文和英文或者其他语言，总体来说套路是一样的，只是有一些简单的变化转换，本文以英文举例，其中包括文本数据预处理准备、词频与停用词、词袋模型、N-Grams模型、TF-IDF 模型、相似性特征、聚类特征、LDA 主题模型、词嵌入模型 word2vec等内容。

# 文本数据预处理准备 #

**1.安装 NLTK**

# shell 命令安装
    pip install nltk
    
    # 进入python环境
    import nltk 
    nltk.download()

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70]

# 构造一个文本数据集 
    corpus = ['The sky is blue and beautiful.',
              'Love this blue and beautiful sky!',
              'The quick brown fox jumps over the lazy dog.',
              'The brown fox is quick and the blue dog is lazy!',
              'The sky is very blue and the sky is very beautiful today',
              'The dog is lazy but the brown fox is quick!'    
    ]
    labels = ['weather', 'weather', 'animals', 'animals', 'weather', 'animals']
    corpus = np.array(corpus)
    corpus_df = pd.DataFrame({ 'Document': corpus, 
                              'Category': labels})
    corpus_df = corpus_df[['Document', 'Category']]
    corpus_df

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 1]  
**2.词频与停用词**

# 词频与停用词
    wpt = nltk.WordPunctTokenizer()
    stop_words = nltk.corpus.stopwords.words('english')
    print (stop_words)
    def normalize_document(doc):
        # 小写转换并删除特殊字符空格
        doc = re.sub(r'[^a-zA-Z0-9\s]', '', doc, re.I)
        doc = doc.lower()
        doc = doc.strip()
        # 标记文档
        tokens = wpt.tokenize(doc)
        # 从文档中过滤停止词
        filtered_tokens = [token for token in tokens if token not in stop_words]
        # 文档中过滤
        doc = ' '.join(filtered_tokens)
        return doc
    
    normalize_corpus = np.vectorize(normalize_document)
    norm_corpus = normalize_corpus(corpus)
    list(norm_corpus)
    
    >>> ['sky blue beautiful',
        'love blue beautiful sky',
        'quick brown fox jumps lazy dog',
        'brown fox quick blue dog lazy',
        'sky blue sky beautiful today',
        'dog lazy brown fox quick']

**3.词袋模型**

# 用0 1标记单词是否出现
    from sklearn.feature_extraction.text import CountVectorizer
    norm_corpus
    >>> ['sky blue beautiful',
         'love blue beautiful sky',
         'quick brown fox jumps lazy dog',
         'brown fox quick blue dog lazy',
         'sky blue sky beautiful today',
         'dog lazy brown fox quick']
    
    cv = CountVectorizer(min_df=0., max_df=1.)
    cv.fit(norm_corpus)
    cv.get_feature_names()
    >>> ['beautiful', 'blue', 'brown', 'dog', 'fox', 'jumps', 'lazy', 'love', 'quick', 'sky', 'today']
    
    
    cv_matrix = cv.fit_transform(norm_corpus)
    cv_matrix = cv_matrix.toarray()
    cv_matrix
    
    >>> array([[1, 1, 0, 0, 0, 0, 0, 0, 0, 1, 0],
               [1, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0],
               [0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0],
               [0, 1, 1, 1, 1, 0, 1, 0, 1, 0, 0],
               [1, 1, 0, 0, 0, 0, 0, 0, 0, 2, 1],
               [0, 0, 1, 1, 1, 0, 1, 0, 1, 0, 0]], dtype=int64)
    
    vocab = cv.get_feature_names()
    pd.DataFrame(cv_matrix, columns=vocab)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 2]  
**4.N-Grams模型**

bv = CountVectorizer(ngram_range=(2,2))  # 两两组合频率
    bv_matrix = bv.fit_transform(norm_corpus)
    bv_matrix = bv_matrix.toarray()
    vocab = bv.get_feature_names()
    pd.DataFrame(bv_matrix, columns=vocab)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 3]  
**5.TF-IDF 模型**

from sklearn.feature_extraction.text import TfidfVectorizer 
    # 词频 * 文档频数
    tv = TfidfVectorizer(min_df=0., max_df=1., use_idf=True) 
    tv_matrix = tv.fit_transform(norm_corpus)
    tv_matrix = tv_matrix.toarray()
    
    vocab = tv.get_feature_names()
    pd.DataFrame(np.round(tv_matrix, 2), columns=vocab)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 4]

**6.相似性特征**

from sklearn.metrics.pairwise import cosine_similarity
    
    similarity_matrix = cosine_similarity(tv_matrix)
    similarity_df = pd.DataFrame(similarity_matrix)
    similarity_df

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 5]

**7.聚类特征**

from sklearn.cluster import KMeans
    
    km = KMeans(n_clusters=2)
    km.fit_transform(similarity_df)
    cluster_labels = km.labels_
    cluster_labels = pd.DataFrame(cluster_labels, columns=['ClusterLabel'])
    pd.concat([corpus_df, cluster_labels], axis=1)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 6]  
**8.LDA 主题模型**

from sklearn.decomposition import LatentDirichletAllocation
    
    lda = LatentDirichletAllocation(n_components=3, max_iter=100, random_state=42)
    dt_matrix = lda.fit_transform(tv_matrix) # 基于TF-IDF值
    features = pd.DataFrame(dt_matrix, columns=['T1', 'T2', 'T3'])
    features

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 7]

# 主题和词的权重
    
    tt_matrix = lda.components_
    for topic_weights in tt_matrix:
        topic = [(token, weight) for token, weight in zip(vocab, topic_weights)]
        topic = sorted(topic, key=lambda x: -x[1])
        topic = [item for item in topic if item[1] > 0.6]
        print(topic)
    
    >>> [('brown', 1.7273638692668465), ('dog', 1.7273638692668465), ('fox', 1.7273638692668465), ('lazy', 1.7273638692668465), ('quick', 1.7273638692668465), ('jumps', 1.0328325272484777), ('blue', 0.7731573162915626)]
    
    >>> [('sky', 2.264386643135622), ('beautiful', 1.9068269319456903), ('blue', 1.7996282104933266), ('love', 1.148127242397004), ('today', 1.0068251160429935)]

**9.词嵌入模型 word2vec**

from gensim.models import word2vec
    
    wpt = nltk.WordPunctTokenizer()
    tokenized_corpus = [wpt.tokenize(document) for document in norm_corpus]
    
    feature_size = 10    # 词向量维数 
    window_context = 10          # 上下文窗口大小 
    min_word_count = 1   # 最小词频数 
    sample = 1e-3   # 高频词汇的随机降采样的配置阈值
    
    w2v_model = word2vec.Word2Vec(tokenized_corpus, size=feature_size, 
                              window=window_context, min_count = min_word_count,
                              sample=sample)
    w2v_model.wv['sky']
    >>> array([-0.01302878, -0.0081328 ,  0.02701689,  0.03391293,  0.01191998,
               -0.00258705,  0.02996921,  0.01644186, -0.03398509, -0.00690445], dtype=float32)
    
    # 单文本数据中词向量求均值
    def average_word_vectors(words, model, vocabulary, num_features):
        
        feature_vector = np.zeros((num_features,),dtype="float64")
        nwords = 0.
        
        for word in words:
            if word in vocabulary: 
                nwords = nwords + 1.
                feature_vector = np.add(feature_vector, model[word])
        
        if nwords:
            feature_vector = np.divide(feature_vector, nwords)
            
        return feature_vector
        
    def averaged_word_vectorizer(corpus, model, num_features):
        vocabulary = set(model.wv.index2word)
        features = [average_word_vectors(tokenized_sentence, model, vocabulary, num_features)
                        for tokenized_sentence in corpus]
        return np.array(features)
    
    w2v_feature_array = averaged_word_vectorizer(corpus=tokenized_corpus, model=w2v_model,
                                                 num_features=feature_size)
    pd.DataFrame(w2v_feature_array)

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 8]

[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70]: /images/20221005/240116ff929040c29f95e992a70a8197.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 1]: /images/20221005/39dbcc6aa1d34aeeb8dc7ca75897f92e.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 2]: /images/20221005/61cf42bc704c441dbfa4e04e9fcdc458.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 3]: /images/20221005/6751bc4be45442c8a4a0ff33d09801e8.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 4]: /images/20221005/f2806427ccf8460a96a9ad6831daad18.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 5]: /images/20221005/1854fac51e1c4fc9b25eae7bb409c660.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 6]: /images/20221005/9be13bffd4c144ad8a71fb1831c5bcde.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 7]: /images/20221005/48046f7e9ab44929bfb21a8884867cf4.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3FxXzIwMjg4MzI3_size_16_color_FFFFFF_t_70 8]: /images/20221005/61595fb7e0024c6a90b91d502e782fef.png