【阿里天池-医学影像报告异常检测】2 特征工程

╰+攻爆jí腚メ 2022-11-03 00:42 174阅读 0赞

### 目录 ###

*  1 引言
 *  2 步骤
 *   *  2.1 导入工具包
     *  2.1 删除句子中高频词后Text作为一个特征
     *  2.2 统计句子长度作为第二个特征
     *  2.3 label缺失值处理
     *  2.4 ID-TDF算法提取text一列数据的特征
 *  2.5 Label编码
 *  3 Main函数
 *  4 数据格式展示
 *   *  4.1 训练样本集
     *  4.2 训练label集
     *  4.4 测试样本集
 *  5 数据增强（失败）
 *   *  5.1 shuffle和dropout
     *  5.2

# 1 引言 #

代码下载[https://github.com/823316627bandeng/TIANCHI-2021-AI-Compition][https_github.com_823316627bandeng_TIANCHI-2021-AI-Compition]  
本人自己的见解，如果有任何疑问和问题，欢迎私信讨论  
  
目前选取 2 个特征：

*  原本 text部分的字符，去除前两个高频字符，id-tdf提取500维特征
 *  句子长度int

# 2 步骤 #

## 2.1 导入工具包 ##

import pandas as pd
    import matplotlib.pyplot as plt
    import numpy as np
    import seaborn as sns
    from scipy import stats
    from sklearn.preprocessing import MultiLabelBinarizer
    from sklearn.feature_extraction.text import CountVectorizer
    from sklearn.feature_extraction.text import TfidfVectorizer
    from scipy import sparse

## 2.1 删除句子中高频词后Text作为一个特征 ##

# 删除句子中高频词
    def delete_highfrequency_word(data):
        text_Word_frequency = []
        from collections import Counter
        datalen = len(data)
        for i in range(0,datalen):
            one_lines = ''.join(list(data['text'][i][1:-1]))
            textls = one_lines.strip().split(" ")
            all_word_count = Counter(textls)
            all_word_count = sorted(all_word_count.items(), key=lambda d:d[1], reverse = True)
            # 删除693和328两个字符，根据数据分析阶段的词频统计，这两个字符可能是标点符号
            dict_word_count = dict(all_word_count)
            if dict_word_count.get('693') !=None:
                del dict_word_count['693']
            if dict_word_count.get('328') !=None:
                del dict_word_count['328']
            string_top10_high_frequency_word = list(dict_word_count.keys())
            if '' in string_top10_high_frequency_word:
                string_top10_high_frequency_word.remove('')
            list_to_str  = " ".join(string_top10_high_frequency_word)
            text_Word_frequency.append(list_to_str)
        return text_Word_frequency

## 2.2 统计句子长度作为第二个特征 ##

# 统计每行的句子长度并添加到train_data中组成新的一个特征
    def count_text_len(data):
        text_len =[]
        datalen = len(data)
        for i in range(0,datalen):
            one_lines = ''.join(list(data['text'][i]))
            len_text = one_lines.strip().split(" ")
            text_len.append(len(len_text))
        return text_len

## 2.3 label缺失值处理 ##

# label缺失值处理，为空的label设置为字符串17，代表无异常
    # 缺失值填充，空label的填充去17
    def label_fill(data):
        new_code_label =[]
        datalen = len(data)
        for i in range(0,datalen):
            one_lines = ''.join(list(data['label'][i][1:])).strip()
            if one_lines =='':#空label
                new_code_label.append('17')
            else:
                new_code_label.append(one_lines)
    
        return new_code_label

## 2.4 ID-TDF算法提取text一列数据的特征 ##

见main函数中

# 2.5 Label编码 #

举例：  
把2，17编码成\[0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1\]

# label编码，一个编码占一个表格，编码后一个标签有18列
    def label_encode(train_data):
        train_data_label = pd.DataFrame(columns=['label'])
        train_data_label['label'] = label_fill(train_data)
        mlb = MultiLabelBinarizer(classes=['0','1','2','3','4','5','6','7','8','9','10','11','12','13','14','15','16','17'])#17表示无异常
        Ylist = []
        all_train_data_label = pd.DataFrame(columns=['f0','f1', 'f2', 'f3','f4','f5','f6','f7', 'f8', 'f9','f10','f11','f12','f13', 'f14', 'f15','f16','f17'],index=[])
        indexsize = 0
        for i in range(len(train_data_label)):
            templist = train_data_label['label'][i].split()
            # 转二值化编码
            label_code_list = list(mlb.fit_transform([templist])[0])
            # 写入DataFrame
            all_train_data_label.loc[indexsize] =label_code_list
            indexsize = indexsize + 1 
        return all_train_data_label

# 3 Main函数 #

import pandas as pd
    import numpy as np
    from utils import *
    from sklearn.feature_extraction.text import TfidfVectorizer
    
    if __name__ =="__main__":
        test_data = pd.read_csv('./data/track1_round1_testA_20210222.csv',header=None, names=['id', 'text'])
        train_data = pd.read_csv('./data/track1_round1_train_20210222.csv',header=None, names=['id', 'text','label'])
        # 删除高频字符，去除首尾的”|“
        print("正在去除高频字符...")
        train_aug_data = pd.DataFrame()
        train_aug_data['label'] = train_data['label']
        # 删除高频字符：693 328 可能是标点符号
        train_aug_data['text']= delete_highfrequency_word(train_data)
        test_data['text'] = delete_highfrequency_word(test_data)
    
        # 合并，准备提取tfidf特征
        print(train_aug_data.shape, test_data.shape)
        df = pd.concat([train_aug_data, test_data], axis=0, ignore_index=True)
        print("正在提取Text特征...")
        tfidf = TfidfVectorizer(ngram_range=(1, 2),max_features=500)
        tfidf_feature = tfidf.fit_transform(df['text'])
        svd_feature = tfidf_feature.toarray()
        # 拆分出训练集和测试集
        train_data_sample = svd_feature[:-3000]
        test_data_sample = svd_feature[-3000:]
        # 统计句子长度，作为一个特征
        print("训练集-正在计算句子长度...")
        train_sample = pd.DataFrame(train_data_sample)
        train_sample.columns = ['tfidf_'+str(i) for i in train_sample.columns]
        train_sample['textlen'] = count_text_len(train_aug_data)
        print("测试集-正在统计句子长度...")
        test_sample = pd.DataFrame(test_data_sample)
        test_sample.columns = ['tfidf_'+str(i) for i in test_sample.columns]
        test_sample['textlen'] = count_text_len(test_data)
        #label编码
        print("正在label编码")
        label = label_encode(train_aug_data)
        label.to_csv('./data/label.csv',index =False)
        train_sample.to_csv('./data/train_sample.csv',index =False)
        test_sample.to_csv('./data/test_sample.csv',index =False)
        print()

# 4 数据格式展示 #

## 4.1 训练样本集 ##

train_data_sample = pd.read_csv('./data/train_data_sample.csv')
    print(train_data_sample.info())

> <class ‘pandas.core.frame.DataFrame’>  
> RangeIndex: 10000 entries, 0 to 9999  
> Data columns (total 82 columns):  
> Column Non-Null Count Dtype  
> 0 Unnamed: 0 10000 non-null int64  
> 1 tfidf\_1 10000 non-null float64  
> …  
> 500 tfidf\_499 10000 non-null float64  
> 501 textlen 10000 non-null int64

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MzkzNTY5Ng_size_16_color_FFFFFF_t_70_pic_center]

## 4.2 训练label集 ##

train_data_label= pd.read_csv('./data/all_train_data_label.csv')
    print(train_data_label.info())

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MzkzNTY5Ng_size_16_color_FFFFFF_t_70_pic_center 1]

## 4.4 测试样本集 ##

test_data_sample = pd.read_csv('./data/all_test_data_sample.csv')
    print(test_data_sample.info())

> <class ‘pandas.core.frame.DataFrame’>  
> RangeIndex: 3000 entries, 0 to 2999  
> Data columns (total 82 columns):  
> Column Non-Null Count Dtype  
> 0 Unnamed: 0 3000 non-null int64  
> 1 tfidf\_0 3000 non-null float64  
> 500 tfidf\_499 3000 non-null float64  
> …  
> 501 textlen 3000 non-null int64

![在这里插入图片描述][watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MzkzNTY5Ng_size_16_color_FFFFFF_t_70_pic_center 2]

# 5 数据增强（失败） #

尝试过的方法

*  shuffle每个句子中的字符
 *  随机删除某个句子中的一半字符
 *  为了处理样本不均衡，对类别少的样本进行复制，多样本太多的进行删除

注意：以下的数据增强在机器学习算法中，在线下训练提高了准确率，但是在线上准确率反而下降，我认为是训练集和测试集的分布不一致导致。以下实现代码仅仅做记录

## 5.1 shuffle和dropout ##

在去除text中高频词的同时，打乱字符顺序并随机删除一半字符

# 去除text中高频词和数据增强
    def delete_highfrequency_word_augmentation(data):
        text_Word_frequency = []
        from collections import Counter
        datalen = len(data)
        for i in range(0,datalen):
            one_lines = ''.join(list(data['text'][i][1:-1]))
            textls = one_lines.strip().split(" ")
            # 数据增强
            # 打乱字符顺序
            textls = shuffle(textls)
            # 随机删除字符
            randn = random.random()
            if randn >0.5 :
                textls = dropout(textls)
            all_word_count = Counter(textls)
            all_word_count = sorted(all_word_count.items(), key=lambda d:d[1], reverse = True)
            # 删除693和328两个字符，根据数据分析阶段的词频统计，这两个字符可能是标点符号
            dict_word_count = dict(all_word_count)
            if dict_word_count.get('693') !=None:
                del dict_word_count['693']
            if dict_word_count.get('328') !=None:
                del dict_word_count['328']
            string_top10_high_frequency_word = list(dict_word_count.keys())
            if '' in string_top10_high_frequency_word:
                string_top10_high_frequency_word.remove('')
            list_to_str  = " ".join(string_top10_high_frequency_word)
            text_Word_frequency.append(list_to_str)
        return text_Word_frequency
    def shuffle(textls):
            shuffle_text = np.random.permutation(textls)
            return shuffle_text
    
    def dropout(textls, p=0.5):
        # random delete some text
        length = len(textls)
        indexs = random.sample(range(length),int(p*length))
        newls = []
        for i, val in enumerate(textls):
            if i not in indexs:
                newls.append(val)
        return newls

## 5.2 ##

对label包含3\\5\\610\\12\\13\\16\\14\\9的样本进行复制一定的量，并删除包含类别15的最多样本的量

def data_expand(train_data):
        train_data_label = pd.DataFrame()
        train_data_label['text'] = train_data['text']
        train_data_label['label'] = label_fill(train_data)
        coutn = 600
        for i in range(len(train_data_label)):
            templist = train_data_label['label'][i].split()
            ''' 判断标签中是否包含以下标签，倍数表示复制样本倍数，作为数据增强的一种 3 3倍 5 3倍 6 5 倍 10 3 倍 12 3倍 16 2 倍 14 2倍 9 2倍 '''
            # key是标签类别，value是复制的次数
            ndict = { '3':3,'5':3,'6':5,'10':3,'12':3,'13':3,'16':2,'14':2,'9':2}
            for key, value in ndict.items():
                # 上采样
                if key in templist:
                    temp_df = train_data_label.iloc[i]
                    for t in range(value):
                          train_data_label.loc[train_data_label.shape[0]] = temp_df
        # 下采样
        for i in range(len(train_data_label)):
            templist = train_data_label['label'][i].split()   
            if ('15' in templist)and (coutn<=1668):
                train_data_label.drop([i],inplace=True)
                coutn +=1
        return train_data_label

[https_github.com_823316627bandeng_TIANCHI-2021-AI-Compition]: https://github.com/823316627bandeng/TIANCHI-2021-AI-Compition
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MzkzNTY5Ng_size_16_color_FFFFFF_t_70_pic_center]: /images/20221024/33c26461dde54e3dbe7f47777484c114.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MzkzNTY5Ng_size_16_color_FFFFFF_t_70_pic_center 1]: /images/20221024/29e0ebd599fd4c898e6ccccbccc00fe6.png
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9ibG9nLmNzZG4ubmV0L3dlaXhpbl80MzkzNTY5Ng_size_16_color_FFFFFF_t_70_pic_center 2]: /images/20221024/9bd591cb133a4850b130ca5da0b7a9bd.png