NLP-预训练模型-2018-Bert-解析：BertForMaskedLM

梦里梦外; 2023-10-01 20:21 76阅读 0赞 
  1. import numpy as np
  2. import torch
  3. import torch.nn as nn
  4. from transformers import BertTokenizer, BertForMaskedLM
  6. # Load pre-trained model (weights)
  7. with torch.no_grad():
  8.     # Load pre-trained model tokenizer (vocabulary)
  9.     tokenizer = BertTokenizer.from_pretrained(r'D:\Pretrained_Model\bert-base-chinese')
  10.     model = BertForMaskedLM.from_pretrained(r'D:\Pretrained_Model\bert-base-chinese')
  11.     model.eval()
  13.     sentence = "我不会忘记和你一起奋斗的时光。"
  14.     tokenize_input = tokenizer.tokenize(sentence)
  15.     print('tokenize_input = ', tokenize_input)
  17.     tensor_input = torch.tensor([tokenizer.convert_tokens_to_ids(tokenize_input)])
  18.     sen_len = len(tokenize_input)
  19.     sentence_loss = 0.
  21.     for idx, word in enumerate(tokenize_input):
  22.         print('\n\n idx = {0}'.format(idx))
  23.         # add mask to i-th character of the sentence
  24.         tokenize_input[idx] = '[MASK]'
  25.         mask_input = torch.tensor([tokenizer.convert_tokens_to_ids(tokenize_input)])
  26.         print('\t mask_input = {0}'.format(mask_input))
  28.         output = model(mask_input)
  29.         print('\n\t output = {0}'.format(output))
  31.         prediction_scores = output[0]
  32.         print('\n\t prediction_scores = output[0] = {0}'.format(prediction_scores))
  34.         softmax = nn.Softmax(dim=0)
  35.         ps = softmax(prediction_scores[0, idx]).log()
  36.         print('\n\t ps = {0}'.format(ps))
  38.         word_loss = ps[tensor_input[0, idx]]
  39.         print('\n\t word_loss = {0}'.format(word_loss))
  41.         sentence_loss += word_loss.item()
  42.         tokenize_input[idx] = word
  44.     ppl = np.exp(-sentence_loss / sen_len)
  46.     print("sentence_loss = {0}；ppl = {1}".format(sentence_loss, ppl))

打印结果：

  1. tokenize_input =  ['我', '不', '会', '忘', '记', '和', '你', '一', '起', '奋', '斗', '的', '时', '光', '。']
  4.  idx = 0
  5.      mask_input = tensor([[ 103,  679,  833, 2563, 6381, 1469,  872,  671, 6629, 1939, 3159, 4638,
  6.          3198, 1045,  511]])
  8.      output = MaskedLMOutput(loss=None, logits=tensor([[[-10.0067,  -9.9702, -10.3403,  ...,  -7.0367,  -7.9918,  -7.8884],
  9.          [ -8.9250,  -8.6627,  -8.8329,  ...,  -5.6988,  -5.0543,  -7.3196],
  10.          [-17.5815, -16.8282, -17.5551,  ..., -11.2575,  -8.4464, -15.8063],
  11.          ...,
  12.          [-17.7271, -17.4097, -18.3814,  ..., -12.5380, -14.9620, -13.0537],
  13.          [-14.8090, -15.5407, -14.8516,  ...,  -9.6344,  -8.9355, -11.3215],
  14.          [-10.2498, -10.0447, -10.2479,  ...,  -5.7584,  -4.9482,  -5.1695]]]), hidden_states=None, attentions=None)
  16.      prediction_scores = output[0] = tensor([[[-10.0067,  -9.9702, -10.3403,  ...,  -7.0367,  -7.9918,  -7.8884],
  17.          [ -8.9250,  -8.6627,  -8.8329,  ...,  -5.6988,  -5.0543,  -7.3196],
  18.          [-17.5815, -16.8282, -17.5551,  ..., -11.2575,  -8.4464, -15.8063],
  19.          ...,
  20.          [-17.7271, -17.4097, -18.3814,  ..., -12.5380, -14.9620, -13.0537],
  21.          [-14.8090, -15.5407, -14.8516,  ...,  -9.6344,  -8.9355, -11.3215],
  22.          [-10.2498, -10.0447, -10.2479,  ...,  -5.7584,  -4.9482,  -5.1695]]])
  24.      ps = tensor([-20.2204, -20.1840, -20.5541,  ..., -17.2505, -18.2055, -18.1022])
  26.      word_loss = -4.207489013671875
  29.  idx = 1
  30.      mask_input = tensor([[2769,  103,  833, 2563, 6381, 1469,  872,  671, 6629, 1939, 3159, 4638,
  31.          3198, 1045,  511]])
  33.      output = MaskedLMOutput(loss=None, logits=tensor([[[-14.2572, -13.5664, -13.7818,  ..., -12.4880, -11.4941, -11.0043],
  34.          [-13.3572, -12.7593, -13.1295,  ...,  -8.9165,  -6.9501,  -8.0928],
  35.          [-18.3267, -17.2391, -16.6626,  ...,  -9.1351,  -8.5136, -10.8610],
  36.          ...,
  37.          [-16.8631, -15.9635, -16.3637,  ..., -11.8876, -12.6025, -10.4363],
  38.          [-14.1836, -14.0044, -13.6275,  ...,  -9.0348, -10.7950,  -9.2346],
  39.          [-16.2714, -15.7472, -15.5543,  ...,  -9.3256,  -9.7824,  -7.4806]]]), hidden_states=None, attentions=None)
  41.      prediction_scores = output[0] = tensor([[[-14.2572, -13.5664, -13.7818,  ..., -12.4880, -11.4941, -11.0043],
  42.          [-13.3572, -12.7593, -13.1295,  ...,  -8.9165,  -6.9501,  -8.0928],
  43.          [-18.3267, -17.2391, -16.6626,  ...,  -9.1351,  -8.5136, -10.8610],
  44.          ...,
  45.          [-16.8631, -15.9635, -16.3637,  ..., -11.8876, -12.6025, -10.4363],
  46.          [-14.1836, -14.0044, -13.6275,  ...,  -9.0348, -10.7950,  -9.2346],
  47.          [-16.2714, -15.7472, -15.5543,  ...,  -9.3256,  -9.7824,  -7.4806]]])
  49.      ps = tensor([-27.0073, -26.4094, -26.7796,  ..., -22.5666, -20.6002, -21.7429])
  51.      word_loss = -3.4179904460906982
  54.  idx = 2
  55.      mask_input = tensor([[2769,  679,  103, 2563, 6381, 1469,  872,  671, 6629, 1939, 3159, 4638,
  56.          3198, 1045,  511]])
  58.      output = MaskedLMOutput(loss=None, logits=tensor([[[-10.4260,  -9.7421, -10.0949,  ...,  -9.1981,  -9.3232,  -9.0737],
  59.          [-11.1497, -10.3329, -10.3952,  ...,  -6.6423,  -5.8855,  -7.4425],
  60.          [-10.2441,  -9.8596, -10.0538,  ...,  -6.8899,  -6.3872,  -7.1557],
  61.          ...,
  62.          [-14.8344, -13.9255, -14.6416,  ..., -11.8463, -11.3034,  -9.4505],
  63.          [-13.0585, -12.7334, -12.5315,  ...,  -9.1430,  -9.0249,  -8.6625],
  64.          [-10.8999, -10.1885, -10.4381,  ...,  -6.9490,  -6.5864,  -5.2088]]]), hidden_states=None, attentions=None)
  66.      prediction_scores = output[0] = tensor([[[-10.4260,  -9.7421, -10.0949,  ...,  -9.1981,  -9.3232,  -9.0737],
  67.          [-11.1497, -10.3329, -10.3952,  ...,  -6.6423,  -5.8855,  -7.4425],
  68.          [-10.2441,  -9.8596, -10.0538,  ...,  -6.8899,  -6.3872,  -7.1557],
  69.          ...,
  70.          [-14.8344, -13.9255, -14.6416,  ..., -11.8463, -11.3034,  -9.4505],
  71.          [-13.0585, -12.7334, -12.5315,  ...,  -9.1430,  -9.0249,  -8.6625],
  72.          [-10.8999, -10.1885, -10.4381,  ...,  -6.9490,  -6.5864,  -5.2088]]])
  74.      ps = tensor([-23.9556, -23.5712, -23.7654,  ..., -20.6015, -20.0987, -20.8673])
  76.      word_loss = -3.0778353214263916
  79.  idx = 3
  80.      mask_input = tensor([[2769,  679,  833,  103, 6381, 1469,  872,  671, 6629, 1939, 3159, 4638,
  81.          3198, 1045,  511]])
  83.      output = MaskedLMOutput(loss=None, logits=tensor([[[-11.1854, -10.8186, -10.8980,  ..., -10.0304,  -6.8312, -10.1228],
  84.          [-18.3292, -17.1635, -18.1168,  ..., -12.8976,  -6.5055, -10.3133],
  85.          [-18.9977, -17.6461, -18.6712,  ..., -12.0834,  -9.4692, -13.3222],
  86.          ...,
  87.          [-15.9868, -15.1038, -15.7956,  ..., -11.8385,  -8.8921, -11.2440],
  88.          [-13.2753, -13.0012, -12.8868,  ...,  -8.5294,  -7.7151,  -9.6861],
  89.          [-14.0791, -13.6179, -13.8650,  ...,  -9.9380,  -8.0259,  -6.6505]]]), hidden_states=None, attentions=None)
  91.      prediction_scores = output[0] = tensor([[[-11.1854, -10.8186, -10.8980,  ..., -10.0304,  -6.8312, -10.1228],
  92.          [-18.3292, -17.1635, -18.1168,  ..., -12.8976,  -6.5055, -10.3133],
  93.          [-18.9977, -17.6461, -18.6712,  ..., -12.0834,  -9.4692, -13.3222],
  94.          ...,
  95.          [-15.9868, -15.1038, -15.7956,  ..., -11.8385,  -8.8921, -11.2440],
  96.          [-13.2753, -13.0012, -12.8868,  ...,  -8.5294,  -7.7151,  -9.6861],
  97.          [-14.0791, -13.6179, -13.8650,  ...,  -9.9380,  -8.0259,  -6.6505]]])
  99.      ps = tensor([-28.6803, -28.3364, -28.7086,  ..., -26.4609, -23.3448, -25.8600])
  101.      word_loss = -0.024608036503195763
  104.  idx = 4
  105.      mask_input = tensor([[2769,  679,  833, 2563,  103, 1469,  872,  671, 6629, 1939, 3159, 4638,
  106.          3198, 1045,  511]])
  108.      output = MaskedLMOutput(loss=None, logits=tensor([[[-11.6949, -11.0920, -11.4218,  ..., -10.2302,  -9.3920, -10.9836],
  109.          [-18.6331, -17.9585, -18.3607,  ..., -12.7316, -10.2360, -14.0741],
  110.          [-19.6247, -18.4559, -19.2653,  ..., -12.6368, -11.0657, -15.6243],
  111.          ...,
  112.          [-15.9810, -15.1353, -15.9852,  ..., -12.4308, -12.2341, -10.9428],
  113.          [-13.4082, -13.1908, -13.3454,  ..., -10.0117, -10.6251, -10.7604],
  114.          [-13.8807, -13.1495, -13.6315,  ...,  -9.3678,  -9.9106,  -7.1275]]]), hidden_states=None, attentions=None)
  116.      prediction_scores = output[0] = tensor([[[-11.6949, -11.0920, -11.4218,  ..., -10.2302,  -9.3920, -10.9836],
  117.          [-18.6331, -17.9585, -18.3607,  ..., -12.7316, -10.2360, -14.0741],
  118.          [-19.6247, -18.4559, -19.2653,  ..., -12.6368, -11.0657, -15.6243],
  119.          ...,
  120.          [-15.9810, -15.1353, -15.9852,  ..., -12.4308, -12.2341, -10.9428],
  121.          [-13.4082, -13.1908, -13.3454,  ..., -10.0117, -10.6251, -10.7604],
  122.          [-13.8807, -13.1495, -13.6315,  ...,  -9.3678,  -9.9106,  -7.1275]]])
  124.      ps = tensor([-30.6680, -30.0711, -30.5083,  ..., -28.1964, -25.7133, -29.4577])
  126.      word_loss = -0.021782301366329193
  129.  idx = 5
  130.      mask_input = tensor([[2769,  679,  833, 2563, 6381,  103,  872,  671, 6629, 1939, 3159, 4638,
  131.          3198, 1045,  511]])
  133.      output = MaskedLMOutput(loss=None, logits=tensor([[[-10.8215, -10.1308, -10.5400,  ...,  -9.4374,  -9.1841,  -9.7690],
  134.          [-16.6464, -15.7021, -16.0986,  ...,  -9.1416,  -7.5447,  -8.9926],
  135.          [-18.4551, -17.0224, -17.3103,  ...,  -8.7594,  -8.8654, -10.6732],
  136.          ...,
  137.          [-14.8322, -13.5759, -14.5636,  ..., -10.8961, -10.6665,  -8.9241],
  138.          [-12.3797, -11.8117, -11.9058,  ...,  -8.7238,  -9.1733,  -9.1059],
  139.          [-12.6140, -11.4767, -11.6919,  ...,  -8.0748,  -9.4955,  -5.7950]]]), hidden_states=None, attentions=None)
  141.      prediction_scores = output[0] = tensor([[[-10.8215, -10.1308, -10.5400,  ...,  -9.4374,  -9.1841,  -9.7690],
  142.          [-16.6464, -15.7021, -16.0986,  ...,  -9.1416,  -7.5447,  -8.9926],
  143.          [-18.4551, -17.0224, -17.3103,  ...,  -8.7594,  -8.8654, -10.6732],
  144.          ...,
  145.          [-14.8322, -13.5759, -14.5636,  ..., -10.8961, -10.6665,  -8.9241],
  146.          [-12.3797, -11.8117, -11.9058,  ...,  -8.7238,  -9.1733,  -9.1059],
  147.          [-12.6140, -11.4767, -11.6919,  ...,  -8.0748,  -9.4955,  -5.7950]]])
  149.      ps = tensor([-20.0339, -19.5133, -20.0343,  ..., -17.0866, -17.4351, -15.4161])
  151.      word_loss = -2.464529037475586
  154.  idx = 6
  155.      mask_input = tensor([[2769,  679,  833, 2563, 6381, 1469,  103,  671, 6629, 1939, 3159, 4638,
  156.          3198, 1045,  511]])
  158.      output = MaskedLMOutput(loss=None, logits=tensor([[[-11.6927, -10.7244, -10.8993,  ...,  -8.0539,  -8.4719,  -9.0431],
  159.          [-14.6502, -14.0066, -14.4193,  ...,  -7.7190,  -5.6522,  -8.8189],
  160.          [-17.8192, -16.0978, -17.0802,  ...,  -8.5008,  -7.9125, -11.5379],
  161.          ...,
  162.          [-15.0797, -14.0576, -14.8092,  ..., -10.5593, -11.1677,  -9.6744],
  163.          [-12.6444, -12.2899, -12.1446,  ...,  -8.7772,  -9.4889,  -9.6838],
  164.          [-11.8326, -11.0902, -11.1104,  ...,  -7.6406,  -8.1461,  -6.2924]]]), hidden_states=None, attentions=None)
  166.      prediction_scores = output[0] = tensor([[[-11.6927, -10.7244, -10.8993,  ...,  -8.0539,  -8.4719,  -9.0431],
  167.          [-14.6502, -14.0066, -14.4193,  ...,  -7.7190,  -5.6522,  -8.8189],
  168.          [-17.8192, -16.0978, -17.0802,  ...,  -8.5008,  -7.9125, -11.5379],
  169.          ...,
  170.          [-15.0797, -14.0576, -14.8092,  ..., -10.5593, -11.1677,  -9.6744],
  171.          [-12.6444, -12.2899, -12.1446,  ...,  -8.7772,  -9.4889,  -9.6838],
  172.          [-11.8326, -11.0902, -11.1104,  ...,  -7.6406,  -8.1461,  -6.2924]]])
  174.      ps = tensor([-17.8420, -17.7343, -17.7814,  ..., -15.6324, -16.8942, -15.6699])
  176.      word_loss = -3.217534065246582
  179.  idx = 7
  180.      mask_input = tensor([[2769,  679,  833, 2563, 6381, 1469,  872,  103, 6629, 1939, 3159, 4638,
  181.          3198, 1045,  511]])
  183.      output = MaskedLMOutput(loss=None, logits=tensor([[[-12.0652, -11.1372, -11.7658,  ..., -10.3255,  -9.5978, -10.2930],
  184.          [-17.4623, -16.3227, -17.0211,  ..., -10.0448,  -8.8320, -11.6701],
  185.          [-19.7825, -18.2467, -18.9617,  ..., -10.4417, -10.0575, -13.2705],
  186.          ...,
  187.          [-16.7194, -15.7009, -16.5568,  ..., -11.9396, -12.9538,  -9.1279],
  188.          [-14.1858, -13.9772, -14.0763,  ...,  -9.9030, -10.4625,  -8.7678],
  189.          [-14.0998, -13.0324, -13.3418,  ...,  -8.7676, -10.0443,  -6.4476]]]), hidden_states=None, attentions=None)
  191.      prediction_scores = output[0] = tensor([[[-12.0652, -11.1372, -11.7658,  ..., -10.3255,  -9.5978, -10.2930],
  192.          [-17.4623, -16.3227, -17.0211,  ..., -10.0448,  -8.8320, -11.6701],
  193.          [-19.7825, -18.2467, -18.9617,  ..., -10.4417, -10.0575, -13.2705],
  194.          ...,
  195.          [-16.7194, -15.7009, -16.5568,  ..., -11.9396, -12.9538,  -9.1279],
  196.          [-14.1858, -13.9772, -14.0763,  ...,  -9.9030, -10.4625,  -8.7678],
  197.          [-14.0998, -13.0324, -13.3418,  ...,  -8.7676, -10.0443,  -6.4476]]])
  199.      ps = tensor([-29.0154, -28.9152, -28.5686,  ..., -23.7333, -25.4041, -24.8862])
  201.      word_loss = -0.006231430917978287
  204.  idx = 8
  205.      mask_input = tensor([[2769,  679,  833, 2563, 6381, 1469,  872,  671,  103, 1939, 3159, 4638,
  206.          3198, 1045,  511]])
  208.      output = MaskedLMOutput(loss=None, logits=tensor([[[-12.3327, -11.7290, -12.1774,  ..., -10.6400,  -9.2812, -10.8762],
  209.          [-17.4025, -16.3325, -17.3093,  ...,  -9.6641,  -8.0054, -10.9477],
  210.          [-19.8157, -18.1812, -19.2325,  ..., -10.3199,  -9.6911, -13.2068],
  211.          ...,
  212.          [-15.4990, -14.1986, -15.4210,  ..., -10.8605, -11.1951,  -9.2175],
  213.          [-13.5214, -13.1154, -13.2580,  ...,  -9.1551,  -8.5442,  -8.5556],
  214.          [-13.9661, -12.7296, -13.4830,  ...,  -7.9905,  -9.4974,  -5.5795]]]), hidden_states=None, attentions=None)
  216.      prediction_scores = output[0] = tensor([[[-12.3327, -11.7290, -12.1774,  ..., -10.6400,  -9.2812, -10.8762],
  217.          [-17.4025, -16.3325, -17.3093,  ...,  -9.6641,  -8.0054, -10.9477],
  218.          [-19.8157, -18.1812, -19.2325,  ..., -10.3199,  -9.6911, -13.2068],
  219.          ...,
  220.          [-15.4990, -14.1986, -15.4210,  ..., -10.8605, -11.1951,  -9.2175],
  221.          [-13.5214, -13.1154, -13.2580,  ...,  -9.1551,  -8.5442,  -8.5556],
  222.          [-13.9661, -12.7296, -13.4830,  ...,  -7.9905,  -9.4974,  -5.5795]]])
  224.      ps = tensor([-26.1031, -25.4673, -25.6910,  ..., -23.7415, -24.6235, -23.6001])
  226.      word_loss = -0.4470815658569336
  229.  idx = 9
  230.      mask_input = tensor([[2769,  679,  833, 2563, 6381, 1469,  872,  671, 6629,  103, 3159, 4638,
  231.          3198, 1045,  511]])
  233.      output = MaskedLMOutput(loss=None, logits=tensor([[[-12.9872, -12.3978, -12.9848,  ..., -11.8125, -12.0875, -12.0079],
  234.          [-17.5210, -16.8555, -17.3870,  ..., -10.8851,  -9.9333, -12.9947],
  235.          [-19.9390, -18.8892, -19.5466,  ..., -12.1456, -11.2809, -13.7224],
  236.          ...,
  237.          [-14.5711, -13.7166, -14.6204,  ..., -10.1978, -11.9384,  -9.0040],
  238.          [-13.0610, -12.8815, -12.9802,  ...,  -9.4830,  -9.4141, -10.5692],
  239.          [-14.2910, -13.4047, -14.0815,  ...,  -8.9679, -11.1636,  -6.8003]]]), hidden_states=None, attentions=None)
  241.      prediction_scores = output[0] = tensor([[[-12.9872, -12.3978, -12.9848,  ..., -11.8125, -12.0875, -12.0079],
  242.          [-17.5210, -16.8555, -17.3870,  ..., -10.8851,  -9.9333, -12.9947],
  243.          [-19.9390, -18.8892, -19.5466,  ..., -12.1456, -11.2809, -13.7224],
  244.          ...,
  245.          [-14.5711, -13.7166, -14.6204,  ..., -10.1978, -11.9384,  -9.0040],
  246.          [-13.0610, -12.8815, -12.9802,  ...,  -9.4830,  -9.4141, -10.5692],
  247.          [-14.2910, -13.4047, -14.0815,  ...,  -8.9679, -11.1636,  -6.8003]]])
  249.      ps = tensor([-23.7871, -23.3177, -23.7469,  ..., -19.6898, -21.1636, -19.3445])
  251.      word_loss = -1.905866265296936
  254.  idx = 10
  255.      mask_input = tensor([[2769,  679,  833, 2563, 6381, 1469,  872,  671, 6629, 1939,  103, 4638,
  256.          3198, 1045,  511]])
  258.      output = MaskedLMOutput(loss=None, logits=tensor([[[-10.2435,  -9.4682,  -9.9029,  ...,  -8.6173,  -7.9944,  -9.5463],
  259.          [-14.7155, -14.1531, -14.7035,  ...,  -7.7060,  -7.0066,  -8.7167],
  260.          [-17.8262, -16.8357, -17.2724,  ...,  -9.3416,  -9.6015, -11.3678],
  261.          ...,
  262.          [-13.5025, -12.6059, -13.4680,  ...,  -9.6887, -10.2040,  -7.5718],
  263.          [-11.8572, -11.8200, -11.6956,  ...,  -8.0838,  -8.2098,  -7.9838],
  264.          [-11.4906, -10.7753, -11.1489,  ...,  -6.4764,  -8.7700,  -4.7994]]]), hidden_states=None, attentions=None)
  266.      prediction_scores = output[0] = tensor([[[-10.2435,  -9.4682,  -9.9029,  ...,  -8.6173,  -7.9944,  -9.5463],
  267.          [-14.7155, -14.1531, -14.7035,  ...,  -7.7060,  -7.0066,  -8.7167],
  268.          [-17.8262, -16.8357, -17.2724,  ...,  -9.3416,  -9.6015, -11.3678],
  269.          ...,
  270.          [-13.5025, -12.6059, -13.4680,  ...,  -9.6887, -10.2040,  -7.5718],
  271.          [-11.8572, -11.8200, -11.6956,  ...,  -8.0838,  -8.2098,  -7.9838],
  272.          [-11.4906, -10.7753, -11.1489,  ...,  -6.4764,  -8.7700,  -4.7994]]])
  274.      ps = tensor([-23.3028, -23.2676, -24.0384,  ..., -20.8967, -21.3373, -20.7125])
  276.      word_loss = -0.3310864269733429
  279.  idx = 11
  280.      mask_input = tensor([[2769,  679,  833, 2563, 6381, 1469,  872,  671, 6629, 1939, 3159,  103,
  281.          3198, 1045,  511]])
  283.      output = MaskedLMOutput(loss=None, logits=tensor([[[-11.1587, -10.6746, -11.6326,  ...,  -9.9938,  -8.8795, -10.4635],
  284.          [-14.6800, -14.1649, -14.6931,  ...,  -7.1853,  -6.1263, -11.4231],
  285.          [-17.5996, -16.4610, -17.1693,  ...,  -7.9229,  -7.0681, -13.4018],
  286.          ...,
  287.          [-13.6107, -12.4848, -13.5183,  ...,  -9.4305,  -9.1442,  -7.4951],
  288.          [-11.5701, -11.2959, -11.3109,  ...,  -7.2745,  -7.0823,  -8.4521],
  289.          [-13.5606, -12.9446, -13.3137,  ...,  -8.5220,  -9.7932,  -7.7482]]]), hidden_states=None, attentions=None)
  291.      prediction_scores = output[0] = tensor([[[-11.1587, -10.6746, -11.6326,  ...,  -9.9938,  -8.8795, -10.4635],
  292.          [-14.6800, -14.1649, -14.6931,  ...,  -7.1853,  -6.1263, -11.4231],
  293.          [-17.5996, -16.4610, -17.1693,  ...,  -7.9229,  -7.0681, -13.4018],
  294.          ...,
  295.          [-13.6107, -12.4848, -13.5183,  ...,  -9.4305,  -9.1442,  -7.4951],
  296.          [-11.5701, -11.2959, -11.3109,  ...,  -7.2745,  -7.0823,  -8.4521],
  297.          [-13.5606, -12.9446, -13.3137,  ...,  -8.5220,  -9.7932,  -7.7482]]])
  299.      ps = tensor([-24.5581, -24.6442, -24.8213,  ..., -21.0443, -21.8916, -20.5020])
  301.      word_loss = -0.0409548319876194
  304.  idx = 12
  305.      mask_input = tensor([[2769,  679,  833, 2563, 6381, 1469,  872,  671, 6629, 1939, 3159, 4638,
  306.           103, 1045,  511]])
  308.      output = MaskedLMOutput(loss=None, logits=tensor([[[-11.2262, -11.1173, -11.6287,  ..., -10.3565,  -9.4391, -11.4225],
  309.          [-15.2089, -14.6585, -15.3925,  ...,  -7.9105,  -6.7598, -10.2716],
  310.          [-17.7514, -16.8604, -17.4242,  ...,  -8.0904,  -8.6169, -12.3799],
  311.          ...,
  312.          [-11.9148, -11.5928, -12.1447,  ...,  -7.0739,  -9.0568,  -7.8991],
  313.          [-10.9299, -10.9160, -10.9438,  ...,  -5.0096,  -7.1774,  -7.3603],
  314.          [-14.6292, -14.3548, -14.3348,  ...,  -6.8946, -10.3034,  -8.7604]]]), hidden_states=None, attentions=None)
  316.      prediction_scores = output[0] = tensor([[[-11.2262, -11.1173, -11.6287,  ..., -10.3565,  -9.4391, -11.4225],
  317.          [-15.2089, -14.6585, -15.3925,  ...,  -7.9105,  -6.7598, -10.2716],
  318.          [-17.7514, -16.8604, -17.4242,  ...,  -8.0904,  -8.6169, -12.3799],
  319.          ...,
  320.          [-11.9148, -11.5928, -12.1447,  ...,  -7.0739,  -9.0568,  -7.8991],
  321.          [-10.9299, -10.9160, -10.9438,  ...,  -5.0096,  -7.1774,  -7.3603],
  322.          [-14.6292, -14.3548, -14.3348,  ...,  -6.8946, -10.3034,  -8.7604]]])
  324.      ps = tensor([-26.6420, -26.3200, -26.8719,  ..., -21.8011, -23.7840, -22.6264])
  326.      word_loss = -0.2741313576698303
  329.  idx = 13
  330.      mask_input = tensor([[2769,  679,  833, 2563, 6381, 1469,  872,  671, 6629, 1939, 3159, 4638,
  331.          3198,  103,  511]])
  333.      output = MaskedLMOutput(loss=None, logits=tensor([[[-12.9708, -12.1014, -12.6502,  ..., -10.6858, -10.9495, -11.6393],
  334.          [-17.4693, -16.4352, -17.2923,  ..., -10.1345,  -9.2979, -12.3043],
  335.          [-19.2976, -17.8839, -18.8252,  ..., -11.4233, -10.9146, -13.9556],
  336.          ...,
  337.          [-14.2439, -13.8837, -14.3827,  ..., -10.8131,  -9.7626, -10.4449],
  338.          [-11.0731, -11.4156, -11.2104,  ...,  -8.5579,  -9.0104,  -8.7935],
  339.          [-13.5802, -13.1632, -13.3280,  ...,  -9.2640, -10.9600,  -8.3216]]]), hidden_states=None, attentions=None)
  341.      prediction_scores = output[0] = tensor([[[-12.9708, -12.1014, -12.6502,  ..., -10.6858, -10.9495, -11.6393],
  342.          [-17.4693, -16.4352, -17.2923,  ..., -10.1345,  -9.2979, -12.3043],
  343.          [-19.2976, -17.8839, -18.8252,  ..., -11.4233, -10.9146, -13.9556],
  344.          ...,
  345.          [-14.2439, -13.8837, -14.3827,  ..., -10.8131,  -9.7626, -10.4449],
  346.          [-11.0731, -11.4156, -11.2104,  ...,  -8.5579,  -9.0104,  -8.7935],
  347.          [-13.5802, -13.1632, -13.3280,  ...,  -9.2640, -10.9600,  -8.3216]]])
  349.      ps = tensor([-26.7180, -27.0605, -26.8553,  ..., -24.2028, -24.6553, -24.4384])
  351.      word_loss = -2.3570048809051514
  354.  idx = 14
  355.      mask_input = tensor([[2769,  679,  833, 2563, 6381, 1469,  872,  671, 6629, 1939, 3159, 4638,
  356.          3198, 1045,  103]])
  358.      output = MaskedLMOutput(loss=None, logits=tensor([[[-11.3928, -10.5590, -11.3276,  ...,  -8.2870,  -7.0281,  -9.3417],
  359.          [-15.1979, -14.2848, -14.9167,  ...,  -8.0477,  -3.3199,  -9.9085],
  360.          [-16.9765, -15.7591, -16.4064,  ...,  -7.3844,  -3.6073, -10.5002],
  361.          ...,
  362.          [-14.3350, -13.5203, -14.7181,  ...,  -9.1939,  -8.4368,  -6.3008],
  363.          [-11.5855, -11.6669, -11.5224,  ...,  -6.1303,  -7.0456,  -5.4713],
  364.          [ -9.3767,  -9.1142,  -9.3964,  ...,  -5.2297,  -5.3290,  -3.2478]]]), hidden_states=None, attentions=None)
  366.      prediction_scores = output[0] = tensor([[[-11.3928, -10.5590, -11.3276,  ...,  -8.2870,  -7.0281,  -9.3417],
  367.          [-15.1979, -14.2848, -14.9167,  ...,  -8.0477,  -3.3199,  -9.9085],
  368.          [-16.9765, -15.7591, -16.4064,  ...,  -7.3844,  -3.6073, -10.5002],
  369.          ...,
  370.          [-14.3350, -13.5203, -14.7181,  ...,  -9.1939,  -8.4368,  -6.3008],
  371.          [-11.5855, -11.6669, -11.5224,  ...,  -6.1303,  -7.0456,  -5.4713],
  372.          [ -9.3767,  -9.1142,  -9.3964,  ...,  -5.2297,  -5.3290,  -3.2478]]])
  374.      ps = tensor([-20.6789, -20.4164, -20.6986,  ..., -16.5319, -16.6312, -14.5500])
  376.      word_loss = -1.3718788623809814
  377. sentence_loss = -23.16600384376943；ppl = 4.685160888290345
  379. Process finished with exit code 0

参考资料：
基于BertForMaskedLM(Language Model) 的数据增强技术

发表评论

表情：
评论列表 （有 0 条评论，76人围观）

还没有评论，来说两句吧...

相关阅读