文本挖掘流程示例

柔光的暖阳◎ 2021-06-11 15:11 333阅读 0赞

![这里写图片描述][70]

import numpy as np
    import pandas as pd
    from contextlib import contextmanager
    from sklearn.feature_extraction.text import TfidfVectorizer
    from scipy.sparse import hstack
    import time
    import re
    import string
    from scipy.sparse import csr_matrix
    from sklearn.preprocessing import MinMaxScaler
    import lightgbm as lgb
    from sklearn.model_selection import KFold
    from sklearn.metrics import roc_auc_score
    import gc
    from collections import defaultdict
    import os
    import psutil
    
    
    # 查看运行内存
    # memory usage
    def cpuStats(disp=""):
        """ @author: RDizzl3 @address: https://www.kaggle.com/rdizzl3"""
        pid = os.getpid()
        py = psutil.Process(pid)
        memoryUse = py.memory_info()[0] / 2. ** 30
        print("%s MEMORY USAGE for PID %10d : %.3f" % (disp, pid, memoryUse))
    
    # 上下文管理
    # context manager
    @contextmanager
    def timer(name):
        """ Taken from Konstantin """
        t0 = time.time()
        yield
        print(f'[{name}] done in {time.time() - t0:.0f} s')
    
    
    # 使用正则匹配将缩略词还原
    # regex match
    # eg. replace "won't" with "will not" and so on
    cont_patterns = [
            (b'US', b'United States'),
            (b'IT', b'Information Technology'),
    # (b"[^a-zA-Z]u[^a-zA-Z]", b"you"),
    # (b"[^a-zA-Z]r[^a-zA-Z]", b"are"),
    # (b"[^a-zA-Z]y[^a-zA-Z]", b"why"),
    # (b"[^a-zA-Z]b4[^a-zA-Z]", b"before"),
            (b'(W|w)on\'t', b'will not'),
            (b'(C|c)an\'t', b'can not'),
            (b'(I|i)\'m', b'i am'),
            (b'(A|a)in\'t', b'is not'),
            (b'(\w+)\'ll', b'\g<1> will'),
            (b'(\w+)n\'t', b'\g<1> not'),
            (b'(\w+)\'ve', b'\g<1> have'),
            (b'(\w+)\'s', b'\g<1> is'),
            (b'(\w+)\'re', b'\g<1> are'),
            (b'(\w+)\'d', b'\g<1> would'),
        ]
    patterns = [(re.compile(regex), repl) for (regex, repl) in cont_patterns]
    
    
    # 数据清洗函数
    # prepare for char n-gram
    def prepare_for_char_n_gram(text):
    
      # 转换大小写
      # 1. lower word
      clean = bytes(text.lower(), encoding="utf-8")
    
      # 除去换行符等
      # 2. remove \n and \t with " " 
      clean = clean.replace(b"\n", b" ")
      clean = clean.replace(b"\t", b" ")
      clean = clean.replace(b"\b", b" ")
      clean = clean.replace(b"\r", b" ")
    
      # 替换缩略词
      # 3. using regex to replace contractions
      for (pattern, repl) in patterns:
          clean = re.sub(pattern, repl, clean)
    
      # 除去标点符号
      # 4. remove punctuation
      # clean = re.sub(b'[^\w^\s]+', b' ', clean)
      exclude = re.compile(b'[%s]' % re.escape(bytes(string.punctuation, encoding='utf-8')))
      clean = b" ".join([exclude.sub(b'', token) for token in clean.split()])
    
      # 除去数字
      # 6. remove numbers
      clean = re.sub(b"\d+", b" ", clean)
    
      # 除去多余空白符
      # 7. remove extra spaces
      clean = re.sub(b'\s+', b' ', clean)
    
      # 将单词转成#包围形式#
      # 5. replace words by words surrounded by # signs 
      # e.g. my name is bond would become #my# #name# #is# #bond# 
      # clean = re.sub(b"([a-z]+)", b"#\g<1>#", clean)
      clean = re.sub(b" ", b"# #", clean)
    
      # 除去结尾空白符
      # remove ending space if any
      clean = re.sub(b'\s+$', b'', clean)
    
      return str(clean, 'utf-8')
    
    # 统计相同匹配的数量 
    # number of same word
    def count_regexp_occ(regexp="", text=None):
        return len(re.findall(regexp, text))
    
    
    # 提取特征并清洗数据
    # extract feature and clean data
    def perform_nlp(df):
        # Check all sorts of content as it may help find toxic comment
    
        # 换行符数量
        # number of \n
        df["ant_slash_n"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\n", x))
    
        # 单词数量
        # number of word
        df["raw_word_len"] = df["comment_text"].apply(lambda x: len(x.split()))
    
        # 字符数量
        # number of char
        df["raw_char_len"] = df["comment_text"].apply(lambda x: len(x))
    
        # 大写字母数量
        # number of A-Z
        df["nb_upper"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[A-Z]", x))
    
        # 检测是否有ip地址
        # Check for IP addresses
        ip_regexp = r" ( ([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]) \.) {3} ([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5]) $"
        # df["has_ip_address"] = df["comment_text"].apply(lambda x: count_regexp_occ(ip_regexp, x))
        # Remove IP address
    
        # 检测是否有fuck等词
        # Number of fuck - f..k contains folk, fork,
        df["nb_fk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[Ff]\S{2}[Kk]", x))
    
        # 检测是否有such等词
        # Number of suck
        df["nb_sk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[Ss]\S{2}[Kk]", x))
        df["nb_dk"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"[dD]ick", x))
        df["nb_you"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\W[Yy]ou\W", x))
        df["nb_mother"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\Wmother\W", x))
        df["nb_nigger"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\Wnigger\W", x))
    
        # start with :
        df["start_with_columns"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"^\:+", x))
    
        # 检测时间戳
        # check for time stamp
        df["has_timestamp"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\d{2}|:\d{2}", x))
        # Remove timestamp
    
        # 检测日期
        # Check for dates 18:44, 8 December 2010
        df["has_date_long"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\D\d{2}:\d{2}, \d{1,2} \w+ \d{4}", x))
        # Remove dates
    
        # 检测短日期
        # Check for date short 8 December 2010
        df["has_date_short"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\D\d{1,2} \w+ \d{4}", x))
    
        # 检测http链接
        # Check for http links
        df["has_http"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"http[s]{0,1}://\S+", x))
        # Remove http links
    
        # 检测email
        # check for mail
        df["has_mail"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\S+\@\w+\.\w+", x))
        # remove mail
    
        # 是否有强调符号
        # number of emphasizing signal
        df["has_emphasize_equal"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\={2}.+\={2}", x))
        df["has_emphasize_quotes"] = df["comment_text"].apply(lambda x: count_regexp_occ(r"\"{4}\S+\"{4}", x))
    
        # 数据清洗
        ####### clean
        ####### clean
        # clean comment_text
        df["clean_comment"] = df["comment_text"].apply(lambda x: prepare_for_char_n_gram(x))
    
        # 清洗后word数量
        # number of clean comments word
        df["clean_word_len"] = df["clean_comment"].apply(lambda x: len(x.split()))
    
        # 清洗后char数量
        # number of clean comments char
        df["clean_char_len"] = df["clean_comment"].apply(lambda x: len(x))
    
        # 清洗后char set数量
        # number and ratio of char set
        df["clean_chars"] = df["clean_comment"].apply(lambda x: len(set(x)))
        df["clean_chars_ratio"] = df["clean_comment"].apply(lambda x: len(set(x))) / df["clean_comment"].apply(lambda x: 1 + min(99, len(x)))
    
        # 清洗后word数量
        # number of clean comments word
        df["clean_word_len"] = df["clean_comment"].apply(lambda x: len(x.split()))
    
        # 清洗后char数量
        # number of clean comments char
        df["clean_char_len"] = df["clean_comment"].apply(lambda x: len(x))
    
    
    # 3-gram char
    def char_analyzer(text):
      tokens = text.split()
      return [token[i: i+3] for token in tokens for i in range(len(token) - 2)]
    
    
    if __name__ == '__main__':
    
        # start garbage collect
        gc.enable()
    
        # label
        class_names = ['toxic', 'severe_toxic', 'obscene', 'threat', 'insult', 'identity_hate']
    
        # read data
        with timer("Reading input files"):
            train = pd.read_csv('../input/train.csv').fillna(' ')
            test = pd.read_csv('../input/test.csv').fillna(' ')
        # CPU state
        cpuStats()
    
        # clean data
        with timer("Performing basic NLP"):
            perform_nlp(train)
            perform_nlp(test)
            # train["ant_slash_n"] = train["comment_text"].apply(lambda x: count_regexp_occ(r"\n", x))
            # test["ant_slash_n"] = test["comment_text"].apply(lambda x: count_regexp_occ(r"\n", x))
            # train['clean_comment'] = train["comment_text"]
            # test['clean_comment'] = test["comment_text"]
        # CPU state
        cpuStats()
    
        # cleaned comments
        train_text = train['clean_comment']
        test_text = test['clean_comment']
        all_text = pd.concat([train_text, test_text])
    
        # MinMaxScaler some feature and Compress sparse mtrix
        with timer("Creating numerical features"):
            num_features = [f_ for f_ in train.columns
                            if f_ not in ["comment_text", "clean_comment", "id", "remaining_chars", 'has_ip_address'] + class_names]
    
            skl = MinMaxScaler()
            train_num_features = csr_matrix(skl.fit_transform(train[num_features]))
            test_num_features = csr_matrix(skl.fit_transform(test[num_features]))
        cpuStats()
    
        # TfidfVectorizer
        with timer("Tfidf on word"):
            word_vectorizer = TfidfVectorizer(
                sublinear_tf=True,
                strip_accents='unicode',
                analyzer='word',
                token_pattern=r'\w{1,}',
                stop_words='english',
                ngram_range=(1, 2),
                max_features=20000)
            word_vectorizer.fit(all_text)
            train_word_features = word_vectorizer.transform(train_text)
            test_word_features = word_vectorizer.transform(test_text)
        del word_vectorizer
        # handle garbage collecollectct
        gc.collect()
        cpuStats()
    
        # Tfidf with n-gram
        with timer("Tfidf on char n_gram"):
            char_vectorizer = TfidfVectorizer(
                sublinear_tf=True,
                strip_accents='unicode',
                tokenizer=char_analyzer,
                analyzer='word',
                ngram_range=(1, 1),
                max_features=50000)
            char_vectorizer.fit(all_text)
            train_char_features = char_vectorizer.transform(train_text)
            test_char_features = char_vectorizer.transform(test_text)
    
        # with timer("Tfidf on char n_gram"):
        # char_vectorizer = TfidfVectorizer(
        # sublinear_tf=True,
        # strip_accents='unicode',
        # analyzer='char',
        # stop_words='english',
        # ngram_range=(1, 5),
        # max_features=20000)
        # char_vectorizer.fit(all_text)
        # train_char_features = char_vectorizer.transform(train_text)
        # test_char_features = char_vectorizer.transform(test_text)
        del char_vectorizer
        gc.collect()
    
        print((train_char_features>0).sum(axis=1).max())
    
        del train_text
        del test_text
        gc.collect()
        cpuStats()
    
    
        # merge train and test features
        with timer("Staking matrices"):
            # three type of train features
            csr_trn = hstack(
                [
                    train_char_features, 
                    train_word_features, 
                    train_num_features
                ]
            ).tocsr()
    
    
            # del train_word_features
            del train_num_features
            del train_char_features
            gc.collect()
    
            # three type of test features
            csr_sub = hstack(
                [
                    test_char_features, 
                    test_word_features, 
                    test_num_features
                ]
            ).tocsr()
    
            # del test_word_features
            del test_num_features
            del test_char_features
            gc.collect()
    
        # get id 
        submission = pd.DataFrame.from_dict({
       'id': test['id']})
        del test
        gc.collect()
    
        # drop some features
        drop_f = [f_ for f_ in train if f_ not in ["id"] + class_names]
        train.drop(drop_f, axis=1, inplace=True)
        gc.collect()
        cpuStats()
    
        # KFold LR Train
        with timer("Scoring LogisticRegression"):
    
            # KFold dataset
            scores = []
            folds = KFold(n_splits=4, shuffle=True, random_state=1)
            lgb_round_dict = defaultdict(int)
            # load dataset
            trn_lgbset = lgb.Dataset(csr_trn, free_raw_data=False) 
    
            cpuStats("LGB Dataset created")
            del csr_trn
            gc.collect()
    
            # single classification lr
            cpuStats("Training csr matrix freed")
            for class_name in class_names:
                print("Class %s scores : " % class_name)
                class_pred = np.zeros(len(train))
                # set label
                train_target = train[class_name]
                trn_lgbset.set_label(train_target.values)
                params = {
                    "objective": "binary",
                    'metric': {
       'auc'},
                    "boosting_type": "gbdt",
                    "verbosity": -1,
                    "num_threads": 4,
                    "bagging_fraction": 0.8,
                    "feature_fraction": 0.8,
                    "learning_rate": 0.1,
                    "num_leaves": 31,
                    "verbose": -1,
                    "min_split_gain": .1,
                    "reg_alpha": .1
                }
                lgb_rounds = 500
    
                for n_fold, (trn_idx, val_idx) in enumerate(folds.split(train, train_target)):
    
                    watchlist = [
                        trn_lgbset.subset(trn_idx), 
                        trn_lgbset.subset(val_idx)
                    ]
                    # Train lgb l1
                    model = lgb.train(
                        params=params,
                        train_set=watchlist[0],
                        num_boost_round=lgb_rounds,
                        valid_sets=watchlist,
                        early_stopping_rounds=50,
                        verbose_eval=0
                    )
                    class_pred[val_idx] = model.predict(trn_lgbset.data[val_idx], num_iteration=model.best_iteration)
                    score = roc_auc_score(train_target.values[val_idx], class_pred[val_idx])
                    lgb_round_dict[class_name] += model.best_iteration
                    print("\t Fold %d : %.6f in %3d rounds" % (n_fold + 1, score, model.best_iteration))
                    cpuStats("End of fold %d" % (n_fold + 1))
                print("full score : %.6f" % roc_auc_score(train_target, class_pred))
                scores.append(roc_auc_score(train_target, class_pred))
                train[class_name + "_oof"] = class_pred
    
            # Save OOF predictions
            train[["id"] + class_names + [f + "_oof" for f in class_names]].to_csv("lvl0_lgbm_clean_oof.csv",
                                                                        index=False,
                                                                        float_format="%.8f")
    
            print('Total CV score is {}'.format(np.mean(scores)))
    
        # LR predict
        with timer("Predicting probabilities"):
            params = {
                    "objective": "binary",
                    'metric': {
       'auc'},
                    "boosting_type": "gbdt",
                    "verbosity": -1,
                    "num_threads": 4,
                    "bagging_fraction": 0.8,
                    "feature_fraction": 0.8,
                    "learning_rate": 0.1, 
                    "num_leaves": 31,
                    "verbose": -1,
                    "min_split_gain": .1,
                    "reg_alpha": .1
                }
    
            for class_name in class_names:
                with timer("Predicting probabilities for %s" % class_name):
                    train_target = train[class_name]
                    trn_lgbset.set_label(train_target.values)
                    # Train lgb
                    model = lgb.train(
                        params=params,
                        train_set=trn_lgbset,
                        num_boost_round=int(lgb_round_dict[class_name] / folds.n_splits)
                    )
                    # predict
                    submission[class_name] = model.predict(csr_sub, num_iteration=model.best_iteration)
    
    # sub file
    submission.to_csv("lvl0_lgbm_clean_sub.csv", index=False, float_format="%.8f")

[70]: /images/20210520/c453cf1b92ee493492f11e8550129265.png