BERT 模型

我会带着你远行 2023-07-02 05:26 12阅读 0赞

# ![20191009191333910.png][] #

# [日萌社][Link 1] #

[人工智能AI：Keras PyTorch MXNet TensorFlow PaddlePaddle 深度学习实战（不定时更新）][AI_Keras PyTorch MXNet TensorFlow PaddlePaddle]

--------------------

# **github下载代码** #

[https://github.com/dragen1860/Deep-Learning-with-TensorFlow-book][https_github.com_dragen1860_Deep-Learning-with-TensorFlow-book]

[https://github.com/dragen1860/TensorFlow-2.x-Tutorials][https_github.com_dragen1860_TensorFlow-2.x-Tutorials]

--------------------

![watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9henVzYS5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70][]

--------------------

# ** \_\_init\_\_.py** #

from .bert import *
    from .loader import *
    from .tokenizer import Tokenizer

--------------------

# **bert.py** #

import math
    import tensorflow as tf
    from tensorflow import keras
    import tensorflow.keras.backend as K
    import numpy as np
    from keras_pos_embd import PositionEmbedding
    from keras_layer_normalization import LayerNormalization
    from keras_transformer import get_encoders
    from keras_transformer import get_custom_objects as get_encoder_custom_objects
    from layers import (get_inputs, get_embedding, TokenEmbedding, EmbeddingSimilarity, Masked, Extract)
    
    
    __all__ = [
        'TOKEN_PAD', 'TOKEN_UNK', 'TOKEN_CLS', 'TOKEN_SEP', 'TOKEN_MASK',
        'gelu', 'get_model', 'get_custom_objects', 'get_base_dict', 'gen_batch_inputs',
    ]
    
    
    TOKEN_PAD = ''  # Token for padding
    TOKEN_UNK = '[UNK]'  # Token for unknown words
    TOKEN_CLS = '[CLS]'  # Token for classification
    TOKEN_SEP = '[SEP]'  # Token for separation
    TOKEN_MASK = '[MASK]'  # Token for masking
    
    
    def gelu(x):
        if K.backend() == 'tensorflow':
            return 0.5 * x * (1.0 + tf.math.erf(x / tf.sqrt(2.0)))
        return 0.5 * x * (1.0 + K.tanh(math.sqrt(2.0 / math.pi) * (x + 0.044715 * K.pow(x, 3))))
    
    
    def get_model(token_num,
                  pos_num=512,
                  seq_len=512,
                  embed_dim=768,
                  transformer_num=12,
                  head_num=12,
                  feed_forward_dim=3072,
                  dropout_rate=0.1,
                  attention_activation=None,
                  feed_forward_activation=gelu,
                  custom_layers=None,
                  training=True,
                  trainable=None,
                  lr=1e-4):
        """Get BERT model.
    
        See: https://arxiv.org/pdf/1810.04805.pdf
    
        :param token_num: Number of tokens.
        :param pos_num: Maximum position.
        :param seq_len: Maximum length of the input sequence or None.
        :param embed_dim: Dimensions of embeddings.
        :param transformer_num: Number of transformers.
        :param head_num: Number of heads in multi-head attention in each transformer.
        :param feed_forward_dim: Dimension of the feed forward layer in each transformer.
        :param dropout_rate: Dropout rate.
        :param attention_activation: Activation for attention layers.
        :param feed_forward_activation: Activation for feed-forward layers.
        :param custom_layers: A function that takes the embedding tensor and returns the tensor after feature extraction.
                              Arguments such as `transformer_num` and `head_num` will be ignored if `custom_layer` is not
                              `None`.
        :param training: The built model will be returned if it is `True`, otherwise the input layers and the last feature
                         extraction layer will be returned.
        :param trainable: Whether the model is trainable.
        :param lr: Learning rate.
        :return: The compiled model.
        """
        if trainable is None:
            trainable = training
        inputs = get_inputs(seq_len=seq_len)
        embed_layer, embed_weights = get_embedding(
            inputs,
            token_num=token_num,
            embed_dim=embed_dim,
            pos_num=pos_num,
            dropout_rate=dropout_rate,
            trainable=trainable,
        )
        transformed = embed_layer
        if custom_layers is not None:
            kwargs = {}
            if keras.utils.generic_utils.has_arg(custom_layers, 'trainable'):
                kwargs['trainable'] = trainable
            transformed = custom_layers(transformed, **kwargs)
        else:
            transformed = get_encoders(
                encoder_num=transformer_num,
                input_layer=transformed,
                head_num=head_num,
                hidden_dim=feed_forward_dim,
                attention_activation=attention_activation,
                feed_forward_activation=feed_forward_activation,
                dropout_rate=dropout_rate,
                trainable=trainable,
            )
        if not training:
            return inputs[:2], transformed
        mlm_dense_layer = keras.layers.Dense(
            units=embed_dim,
            activation=feed_forward_activation,
            trainable=trainable,
            name='MLM-Dense',
        )(transformed)
        mlm_norm_layer = LayerNormalization(name='MLM-Norm')(mlm_dense_layer)
        mlm_pred_layer = EmbeddingSimilarity(name='MLM-Sim')([mlm_norm_layer, embed_weights])
        masked_layer = Masked(name='MLM')([mlm_pred_layer, inputs[-1]])
        extract_layer = Extract(index=0, name='Extract')(transformed)
        nsp_dense_layer = keras.layers.Dense(
            units=embed_dim,
            activation='tanh',
            trainable=trainable,
            name='NSP-Dense',
        )(extract_layer)
        nsp_pred_layer = keras.layers.Dense(
            units=2,
            activation='softmax',
            trainable=trainable,
            name='NSP',
        )(nsp_dense_layer)
        model = keras.models.Model(inputs=inputs, outputs=[masked_layer, nsp_pred_layer])
        model.compile(
            optimizer=keras.optimizers.Adam(lr=lr),
            loss=keras.losses.sparse_categorical_crossentropy,
        )
        return model
    
    
    def get_custom_objects():
        """Get all custom objects for loading saved models."""
        custom_objects = get_encoder_custom_objects()
        custom_objects['PositionEmbedding'] = PositionEmbedding
        custom_objects['TokenEmbedding'] = TokenEmbedding
        custom_objects['EmbeddingSimilarity'] = EmbeddingSimilarity
        custom_objects['Masked'] = Masked
        custom_objects['Extract'] = Extract
        custom_objects['gelu'] = gelu
        return custom_objects
    
    
    def get_base_dict():
        """Get basic dictionary containing special tokens."""
        return {
            TOKEN_PAD: 0,
            TOKEN_UNK: 1,
            TOKEN_CLS: 2,
            TOKEN_SEP: 3,
            TOKEN_MASK: 4,
        }
    
    
    def gen_batch_inputs(sentence_pairs,
                         token_dict,
                         token_list,
                         seq_len=512,
                         mask_rate=0.15,
                         mask_mask_rate=0.8,
                         mask_random_rate=0.1,
                         swap_sentence_rate=0.5,
                         force_mask=True):
        """Generate a batch of inputs and outputs for training.
    
        :param sentence_pairs: A list of pairs containing lists of tokens.
        :param token_dict: The dictionary containing special tokens.
        :param token_list: A list containing all tokens.
        :param seq_len: Length of the sequence.
        :param mask_rate: The rate of choosing a token for prediction.
        :param mask_mask_rate: The rate of replacing the token to `TOKEN_MASK`.
        :param mask_random_rate: The rate of replacing the token to a random word.
        :param swap_sentence_rate: The rate of swapping the second sentences.
        :param force_mask: At least one position will be masked.
        :return: All the inputs and outputs.
        """
        batch_size = len(sentence_pairs)
        base_dict = get_base_dict()
        unknown_index = token_dict[TOKEN_UNK]
        # Generate sentence swapping mapping
        nsp_outputs = np.zeros((batch_size,))
        mapping = {}
        if swap_sentence_rate > 0.0:
            indices = [index for index in range(batch_size) if np.random.random() < swap_sentence_rate]
            mapped = indices[:]
            np.random.shuffle(mapped)
            for i in range(len(mapped)):
                if indices[i] != mapped[i]:
                    nsp_outputs[indices[i]] = 1.0
            mapping = {indices[i]: mapped[i] for i in range(len(indices))}
        # Generate MLM
        token_inputs, segment_inputs, masked_inputs = [], [], []
        mlm_outputs = []
        for i in range(batch_size):
            first, second = sentence_pairs[i][0], sentence_pairs[mapping.get(i, i)][1]
            segment_inputs.append(([0] * (len(first) + 2) + [1] * (seq_len - (len(first) + 2)))[:seq_len])
            tokens = [TOKEN_CLS] + first + [TOKEN_SEP] + second + [TOKEN_SEP]
            tokens = tokens[:seq_len]
            tokens += [TOKEN_PAD] * (seq_len - len(tokens))
            token_input, masked_input, mlm_output = [], [], []
            has_mask = False
            for token in tokens:
                mlm_output.append(token_dict.get(token, unknown_index))
                if token not in base_dict and np.random.random() < mask_rate:
                    has_mask = True
                    masked_input.append(1)
                    r = np.random.random()
                    if r < mask_mask_rate:
                        token_input.append(token_dict[TOKEN_MASK])
                    elif r < mask_mask_rate + mask_random_rate:
                        while True:
                            token = np.random.choice(token_list)
                            if token not in base_dict:
                                token_input.append(token_dict[token])
                                break
                    else:
                        token_input.append(token_dict.get(token, unknown_index))
                else:
                    masked_input.append(0)
                    token_input.append(token_dict.get(token, unknown_index))
            if force_mask and not has_mask:
                masked_input[1] = 1
            token_inputs.append(token_input)
            masked_inputs.append(masked_input)
            mlm_outputs.append(mlm_output)
        inputs = [np.asarray(x) for x in [token_inputs, segment_inputs, masked_inputs]]
        outputs = [np.asarray(np.expand_dims(x, axis=-1)) for x in [mlm_outputs, nsp_outputs]]
        return inputs, outputs

#  #

--------------------

# **loader.py** #

import json
    from tensorflow import keras
    import numpy as np
    import tensorflow as tf
    from .bert import get_model
    
    
    __all__ = [
        'build_model_from_config',
        'load_model_weights_from_checkpoint',
        'load_trained_model_from_checkpoint',
    ]
    
    
    def checkpoint_loader(checkpoint_file):
        def _loader(name):
            return tf.train.load_variable(checkpoint_file, name)
        return _loader
    
    
    def build_model_from_config(config_file,
                                training=False,
                                trainable=None,
                                seq_len=None):
        """Build the model from config file.
    
        :param config_file: The path to the JSON configuration file.
        :param training: If training, the whole model will be returned.
        :param trainable: Whether the model is trainable.
        :param seq_len: If it is not None and it is shorter than the value in the config file, the weights in
                        position embeddings will be sliced to fit the new length.
        :return: model and config
        """
        with open(config_file, 'r') as reader:
            config = json.loads(reader.read())
        if seq_len is not None:
            config['max_position_embeddings'] = min(seq_len, config['max_position_embeddings'])
        if trainable is None:
            trainable = training
        model = get_model(
            token_num=config['vocab_size'],
            pos_num=config['max_position_embeddings'],
            seq_len=config['max_position_embeddings'],
            embed_dim=config['hidden_size'],
            transformer_num=config['num_hidden_layers'],
            head_num=config['num_attention_heads'],
            feed_forward_dim=config['intermediate_size'],
            training=training,
            trainable=trainable,
        )
        if not training:
            inputs, outputs = model
            model = keras.models.Model(inputs=inputs, outputs=outputs)
            model.compile(
                optimizer=keras.optimizers.Adam(),
                loss=keras.losses.sparse_categorical_crossentropy,
            )
        return model, config
    
    
    def load_model_weights_from_checkpoint(model,
                                           config,
                                           checkpoint_file,
                                           training=False):
        """Load trained official model from checkpoint.
    
        :param model: Built keras model.
        :param config: Loaded configuration file.
        :param checkpoint_file: The path to the checkpoint files, should end with '.ckpt'.
        :param training: If training, the whole model will be returned.
                         Otherwise, the MLM and NSP parts will be ignored.
        """
        loader = checkpoint_loader(checkpoint_file)
    
        model.get_layer(name='Embedding-Token').set_weights([
            loader('bert/embeddings/word_embeddings'),
        ])
        model.get_layer(name='Embedding-Position').set_weights([
            loader('bert/embeddings/position_embeddings')[:config['max_position_embeddings'], :],
        ])
        model.get_layer(name='Embedding-Segment').set_weights([
            loader('bert/embeddings/token_type_embeddings'),
        ])
        model.get_layer(name='Embedding-Norm').set_weights([
            loader('bert/embeddings/LayerNorm/gamma'),
            loader('bert/embeddings/LayerNorm/beta'),
        ])
        for i in range(config['num_hidden_layers']):
            model.get_layer(name='Encoder-%d-MultiHeadSelfAttention' % (i + 1)).set_weights([
                loader('bert/encoder/layer_%d/attention/self/query/kernel' % i),
                loader('bert/encoder/layer_%d/attention/self/query/bias' % i),
                loader('bert/encoder/layer_%d/attention/self/key/kernel' % i),
                loader('bert/encoder/layer_%d/attention/self/key/bias' % i),
                loader('bert/encoder/layer_%d/attention/self/value/kernel' % i),
                loader('bert/encoder/layer_%d/attention/self/value/bias' % i),
                loader('bert/encoder/layer_%d/attention/output/dense/kernel' % i),
                loader('bert/encoder/layer_%d/attention/output/dense/bias' % i),
            ])
            model.get_layer(name='Encoder-%d-MultiHeadSelfAttention-Norm' % (i + 1)).set_weights([
                loader('bert/encoder/layer_%d/attention/output/LayerNorm/gamma' % i),
                loader('bert/encoder/layer_%d/attention/output/LayerNorm/beta' % i),
            ])
            model.get_layer(name='Encoder-%d-MultiHeadSelfAttention-Norm' % (i + 1)).set_weights([
                loader('bert/encoder/layer_%d/attention/output/LayerNorm/gamma' % i),
                loader('bert/encoder/layer_%d/attention/output/LayerNorm/beta' % i),
            ])
            model.get_layer(name='Encoder-%d-FeedForward' % (i + 1)).set_weights([
                loader('bert/encoder/layer_%d/intermediate/dense/kernel' % i),
                loader('bert/encoder/layer_%d/intermediate/dense/bias' % i),
                loader('bert/encoder/layer_%d/output/dense/kernel' % i),
                loader('bert/encoder/layer_%d/output/dense/bias' % i),
            ])
            model.get_layer(name='Encoder-%d-FeedForward-Norm' % (i + 1)).set_weights([
                loader('bert/encoder/layer_%d/output/LayerNorm/gamma' % i),
                loader('bert/encoder/layer_%d/output/LayerNorm/beta' % i),
            ])
        if training:
            model.get_layer(name='MLM-Dense').set_weights([
                loader('cls/predictions/transform/dense/kernel'),
                loader('cls/predictions/transform/dense/bias'),
            ])
            model.get_layer(name='MLM-Norm').set_weights([
                loader('cls/predictions/transform/LayerNorm/gamma'),
                loader('cls/predictions/transform/LayerNorm/beta'),
            ])
            model.get_layer(name='MLM-Sim').set_weights([
                loader('cls/predictions/output_bias'),
            ])
            model.get_layer(name='NSP-Dense').set_weights([
                loader('bert/pooler/dense/kernel'),
                loader('bert/pooler/dense/bias'),
            ])
            model.get_layer(name='NSP').set_weights([
                np.transpose(loader('cls/seq_relationship/output_weights')),
                loader('cls/seq_relationship/output_bias'),
            ])
    
    
    def load_trained_model_from_checkpoint(config_file,
                                           checkpoint_file,
                                           training=False,
                                           trainable=None,
                                           seq_len=None):
        """Load trained official model from checkpoint.
    
        :param config_file: The path to the JSON configuration file.
        :param checkpoint_file: The path to the checkpoint files, should end with '.ckpt'.
        :param training: If training, the whole model will be returned.
                         Otherwise, the MLM and NSP parts will be ignored.
        :param trainable: Whether the model is trainable. The default value is the same with `training`.
        :param seq_len: If it is not None and it is shorter than the value in the config file, the weights in
                        position embeddings will be sliced to fit the new length.
        :return: model
        """
        model, config = build_model_from_config(config_file, training=training, trainable=trainable, seq_len=seq_len)
        load_model_weights_from_checkpoint(model, config, checkpoint_file, training=training)
        return model

#  #

--------------------

# **main.py** #

from tensorflow import keras
    from bert import get_base_dict, get_model, gen_batch_inputs
    
    
    # A toy input example
    sentence_pairs = [
        [['all', 'work', 'and', 'no', 'play'], ['makes', 'jack', 'a', 'dull', 'boy']],
        [['from', 'the', 'day', 'forth'], ['my', 'arm', 'changed']],
        [['and', 'a', 'voice', 'echoed'], ['power', 'give', 'me', 'more', 'power']],
    ]
    
    
    # Build token dictionary
    token_dict = get_base_dict()  # A dict that contains some special tokens
    for pairs in sentence_pairs:
        for token in pairs[0] + pairs[1]:
            if token not in token_dict:
                token_dict[token] = len(token_dict)
    token_list = list(token_dict.keys())  # Used for selecting a random word
    
    
    # Build & train the model
    model = get_model(
        token_num=len(token_dict),
        head_num=5,
        transformer_num=12,
        embed_dim=25,
        feed_forward_dim=100,
        seq_len=20,
        pos_num=20,
        dropout_rate=0.05,
    )
    model.summary()
    
    def _generator():
        while True:
            yield gen_batch_inputs(
                sentence_pairs,
                token_dict,
                token_list,
                seq_len=20,
                mask_rate=0.3,
                swap_sentence_rate=1.0,
            )
    
    model.fit_generator(
        generator=_generator(),
        steps_per_epoch=1000,
        epochs=100,
        validation_data=_generator(),
        validation_steps=100,
        callbacks=[
            keras.callbacks.EarlyStopping(monitor='val_loss', patience=5)
        ],
    )
    
    
    # Use the trained model
    inputs, output_layer = get_model(
        token_num=len(token_dict),
        head_num=5,
        transformer_num=12,
        embed_dim=25,
        feed_forward_dim=100,
        seq_len=20,
        pos_num=20,
        dropout_rate=0.05,
        training=False,      # The input layers and output layer will be returned if `training` is `False`
        trainable=False,     # Whether the model is trainable. The default value is the same with `training`
        output_layer_num=4,  # The number of layers whose outputs will be concatenated as a single output.
                             # Only available when `training` is `False`.
    )

#  #

--------------------

# **tokenizer.py** #

import unicodedata
    from .bert import TOKEN_CLS, TOKEN_SEP, TOKEN_UNK
    
    
    class Tokenizer(object):
    
        def __init__(self,
                     token_dict,
                     token_cls=TOKEN_CLS,
                     token_sep=TOKEN_SEP,
                     token_unk=TOKEN_UNK,
                     pad_index=0,
                     cased=False):
            """Initialize tokenizer.
    
            :param token_dict: A dict maps tokens to indices.
            :param token_cls: The token represents classification.
            :param token_sep: The token represents separator.
            :param token_unk: The token represents unknown token.
            :param pad_index: The index to pad.
            :param cased: Whether to keep the case.
            """
            self._token_dict = token_dict
            self._token_cls = token_cls
            self._token_sep = token_sep
            self._token_unk = token_unk
            self._pad_index = pad_index
            self._cased = cased
    
        @staticmethod
        def _truncate(first_tokens, second_tokens=None, max_len=None):
            if max_len is None:
                return
    
            if second_tokens is not None:
                while True:
                    total_len = len(first_tokens) + len(second_tokens)
                    if total_len <= max_len - 3:  # 3 for [CLS] .. tokens_a .. [SEP] .. tokens_b [SEP]
                        break
                    if len(first_tokens) > len(second_tokens):
                        first_tokens.pop()
                    else:
                        second_tokens.pop()
            else:
                del first_tokens[max_len - 2:]  # 2 for [CLS] .. tokens .. [SEP]
    
        def _pack(self, first_tokens, second_tokens=None):
            first_packed_tokens = [self._token_cls] + first_tokens + [self._token_sep]
            if second_tokens is not None:
                second_packed_tokens = second_tokens + [self._token_sep]
                return first_packed_tokens + second_packed_tokens, len(first_packed_tokens), len(second_packed_tokens)
            else:
                return first_packed_tokens, len(first_packed_tokens), 0
    
        def _convert_tokens_to_ids(self, tokens):
            unk_id = self._token_dict.get(self._token_unk)
            return [self._token_dict.get(token, unk_id) for token in tokens]
    
        def tokenize(self, first, second=None):
            first_tokens = self._tokenize(first)
            second_tokens = self._tokenize(second) if second is not None else None
            tokens, _, _ = self._pack(first_tokens, second_tokens)
            return tokens
    
        def encode(self, first, second=None, max_len=None):
            first_tokens = self._tokenize(first)
            second_tokens = self._tokenize(second) if second is not None else None
            self._truncate(first_tokens, second_tokens, max_len)
            tokens, first_len, second_len = self._pack(first_tokens, second_tokens)
    
            token_ids = self._convert_tokens_to_ids(tokens)
            segment_ids = [0] * first_len + [1] * second_len
    
            if max_len is not None:
                pad_len = max_len - first_len - second_len
                token_ids += [self._pad_index] * pad_len
                segment_ids += [0] * pad_len
    
            return token_ids, segment_ids
    
        def _tokenize(self, text):
            if not self._cased:
                text = unicodedata.normalize('NFD', text)
                text = ''.join([ch for ch in text if unicodedata.category(ch) != 'Mn'])
                text = text.lower()
            spaced = ''
            for ch in text:
                if self._is_punctuation(ch) or self._is_cjk_character(ch):
                    spaced += ' ' + ch + ' '
                elif self._is_space(ch):
                    spaced += ' '
                elif ord(ch) == 0 or ord(ch) == 0xfffd or self._is_control(ch):
                    continue
                else:
                    spaced += ch
            tokens = []
            for word in spaced.strip().split():
                tokens += self._word_piece_tokenize(word)
            return tokens
    
        def _word_piece_tokenize(self, word):
            if word in self._token_dict:
                return [word]
            tokens = []
            start, stop = 0, 0
            while start < len(word):
                stop = len(word)
                while stop > start:
                    sub = word[start:stop]
                    if start > 0:
                        sub = '##' + sub
                    if sub in self._token_dict:
                        break
                    stop -= 1
                if start == stop:
                    stop += 1
                tokens.append(sub)
                start = stop
            return tokens
    
        @staticmethod
        def _is_punctuation(ch):
            code = ord(ch)
            return 33 <= code <= 47 or \
                58 <= code <= 64 or \
                91 <= code <= 96 or \
                123 <= code <= 126 or \
                unicodedata.category(ch).startswith('P')
    
        @staticmethod
        def _is_cjk_character(ch):
            code = ord(ch)
            return 0x4E00 <= code <= 0x9FFF or \
                0x3400 <= code <= 0x4DBF or \
                0x20000 <= code <= 0x2A6DF or \
                0x2A700 <= code <= 0x2B73F or \
                0x2B740 <= code <= 0x2B81F or \
                0x2B820 <= code <= 0x2CEAF or \
                0xF900 <= code <= 0xFAFF or \
                0x2F800 <= code <= 0x2FA1F
    
        @staticmethod
        def _is_space(ch):
            return ch == ' ' or ch == '\n' or ch == '\r' or ch == '\t' or \
                unicodedata.category(ch) == 'Zs'
    
        @staticmethod
        def _is_control(ch):
            return unicodedata.category(ch).startswith('C')

[20191009191333910.png]: https://img-blog.csdnimg.cn/20191009191333910.png
[Link 1]: http://www.rimengshe.com/
[AI_Keras PyTorch MXNet TensorFlow PaddlePaddle]: https://blog.csdn.net/zimiao552147572/article/details/88867161
[https_github.com_dragen1860_Deep-Learning-with-TensorFlow-book]: https://github.com/dragen1860/Deep-Learning-with-TensorFlow-book
[https_github.com_dragen1860_TensorFlow-2.x-Tutorials]: https://github.com/dragen1860/TensorFlow-2.x-Tutorials
[watermark_type_ZmFuZ3poZW5naGVpdGk_shadow_10_text_aHR0cHM6Ly9henVzYS5ibG9nLmNzZG4ubmV0_size_16_color_FFFFFF_t_70]: https://img-blog.csdnimg.cn/20200126210156235.png?x-oss-process=image/watermark,type_ZmFuZ3poZW5naGVpdGk,shadow_10,text_aHR0cHM6Ly9henVzYS5ibG9nLmNzZG4ubmV0,size_16,color_FFFFFF,t_70