from transformers import AutoTokenizer

# トークナイザーの準備
tokenizer = AutoTokenizer.from_pretrained('bert-base-cased')

Downloading: 100%
570/570 [00:00<00:00, 13.3kB/s]
Downloading: 100%
213k/213k [00:00<00:00, 563kB/s]
Downloading: 100%
436k/436k [00:00<00:00, 587kB/s]
Downloading: 100%
29.0/29.0 [00:00<00:00, 725B/s]

# 自然言語テキストをテンソルにエンコード
input = tokenizer.encode("Hello, I'm a single sentence!", return_tensors='pt', max_length=512, truncation=True)
print(input)

tensor([[ 101, 8667,  117,  146,  112,  182,  170, 1423, 5650,  106,  102]])

# テンソルを自然言語テキストにデコード
print(tokenizer.decode(input.numpy()[0]))

[CLS] Hello, I'm a single sentence! [SEP]

# 複数の自然言語テキストをテンソルにエンコード
inputs = tokenizer.encode_plus('This is a question', 'This is a context!', return_tensors='pt')
print(inputs)

{'input_ids': tensor([[ 101, 1188, 1110,  170, 2304,  102, 1188, 1110,  170, 5618,  106,  102]]), 
 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1]]), 
 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]])}

# テンソルを複数の自然言語テキストにデコード
print(tokenizer.decode(inputs['input_ids'].numpy()[0]))

[CLS] This is a question [SEP] This is a context! [SEP]

from transformers import pipeline

# MLMのパイプラインの準備
nlp = pipeline('fill-mask')

# マスク位置の単語を予測
print(nlp(f'Once upon a time there was a young lady and an old {nlp.tokenizer.mask_token}.'))

[{'sequence': 'Once upon a time there was a young lady and an old gentleman.', 'score': 0.49178990721702576, 'token': 22164, 'token_str': ' gentleman'}, 
 {'sequence': 'Once upon a time there was a young lady and an old man.', 'score': 0.2817203998565674, 'token': 313, 'token_str': ' man'}, 
 {'sequence': 'Once upon a time there was a young lady and an old boy.', 'score': 0.08439457416534424, 'token': 2143, 'token_str': ' boy'}, 
 {'sequence': 'Once upon a time there was a young lady and an old lady.', 'score': 0.06599652022123337, 'token': 6429, 'token_str': ' lady'}, 
 {'sequence': 'Once upon a time there was a young lady and an old woman.', 'score': 0.01124062854796648, 'token': 693, 'token_str': ' woman'}]

from transformers import pipeline

# テキスト生成のパイプライン
text_generator = pipeline('text-generation')

# 入力テキスト
text = 'Once upon a time there was an old man and an old woman.'

# テキスト生成
print(text_generator(text, max_length=100, min_length=50))

Some weights of GPT2Model were not initialized from the model checkpoint at gpt2 and are newly initialized: ['h.0.attn.masked_bias', 'h.1.attn.masked_bias', 'h.2.attn.masked_bias', 'h.3.attn.masked_bias', 'h.4.attn.masked_bias', 'h.5.attn.masked_bias', 'h.6.attn.masked_bias', 'h.7.attn.masked_bias', 'h.8.attn.masked_bias', 'h.9.attn.masked_bias', 'h.10.attn.masked_bias', 'h.11.attn.masked_bias']
You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.
Setting `pad_token_id` to `eos_token_id`:50256 for open-end generation.
[{'generated_text': 'Once upon a time there was an old man and an old woman. One was a very good maid on whom the women worked while the young men worked in the yard.\n\nThis time in the middle of the night, the young maid ran off the edge of the street, followed by several other girls, and as they approached the old maid came the other to speak to this man. He gave her a call and said he had come for someone. The young maid took the call, and found'}]

from transformers import pipeline

# 要約のパイプラインの準備
summarizer = pipeline('summarization')

# テキスト
text = 'Huggingface Transformers is a deep learning framework provided by Huggingface that specializes in natural language processing. It supports both TensorFlow and PyTorch. You can use deep learning to solve natural language processing tasks such as text classification, question answering, and summary.'

# 推論
print(summarizer(text, max_length=20, min_length=10))

[{'summary_text': ' Huggingface Transformers is a deep learning framework that specializes in natural language processing . It'}]

from transformers import pipeline

# 質疑応答のパイプラインの準備
nlp = pipeline('question-answering')

# コンテキスト
context = 'Huggingface Transformers is a deep learning framework provided by Huggingface that specializes in natural language processing. It supports both TensorFlow and PyTorch. You can use deep learning to solve natural language processing tasks such as text classification, question answering, and summary.'

# 質問
question = 'What is a natural language processing task?'

# 推論
print(nlp(question=question, context=context))

{'score': 0.6564476490020752, 'start': 245, 'end': 297, 'answer': 'text classification, question answering, and summary'}

!pip install transformers[ja]==4.4.2

Collecting transformers[ja]==4.4.2
  Downloading transformers-4.4.2-py3-none-any.whl (2.0 MB)
     |████████████████████████████████| 2.0 MB 13.1 MB/s 
Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers[ja]==4.4.2) (3.0.12)
Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers[ja]==4.4.2) (21.0)
Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers[ja]==4.4.2) (2019.12.20)
Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers[ja]==4.4.2) (4.8.1)
Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers[ja]==4.4.2) (1.19.5)
Collecting tokenizers<0.11,>=0.10.1
  Downloading tokenizers-0.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl (3.3 MB)
     |████████████████████████████████| 3.3 MB 56.6 MB/s 
Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers[ja]==4.4.2) (4.62.2)
Collecting sacremoses
  Downloading sacremoses-0.0.45-py3-none-any.whl (895 kB)
     |████████████████████████████████| 895 kB 55.7 MB/s 
Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers[ja]==4.4.2) (2.23.0)
Collecting fugashi>=1.0
  Downloading fugashi-1.1.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.whl (490 kB)
     |████████████████████████████████| 490 kB 60.5 MB/s 
Collecting ipadic<2.0,>=1.0.0
  Downloading ipadic-1.0.0.tar.gz (13.4 MB)
     |████████████████████████████████| 13.4 MB 212 kB/s 
Collecting unidic>=1.0.2
  Downloading unidic-1.0.3.tar.gz (5.1 kB)
Collecting unidic-lite>=1.0.7
  Downloading unidic-lite-1.0.8.tar.gz (47.4 MB)
     |████████████████████████████████| 47.4 MB 79 kB/s 
Requirement already satisfied: wasabi<1.0.0,>=0.6.0 in /usr/local/lib/python3.7/dist-packages (from unidic>=1.0.2->transformers[ja]==4.4.2) (0.8.2)
Requirement already satisfied: plac<2.0.0,>=1.1.3 in /usr/local/lib/python3.7/dist-packages (from unidic>=1.0.2->transformers[ja]==4.4.2) (1.1.3)
Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers[ja]==4.4.2) (3.0.4)
Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers[ja]==4.4.2) (1.24.3)
Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers[ja]==4.4.2) (2021.5.30)
Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers[ja]==4.4.2) (2.10)
Requirement already satisfied: typing-extensions>=3.6.4 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers[ja]==4.4.2) (3.7.4.3)
Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers[ja]==4.4.2) (3.5.0)
Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers[ja]==4.4.2) (2.4.7)
Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers[ja]==4.4.2) (1.15.0)
Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers[ja]==4.4.2) (1.0.1)
Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers[ja]==4.4.2) (7.1.2)
Building wheels for collected packages: ipadic, unidic, unidic-lite
  Building wheel for ipadic (setup.py) ... done
  Created wheel for ipadic: filename=ipadic-1.0.0-py3-none-any.whl size=13556723 sha256=868f02ff1674a53b19d8180bdf7a9473a57ab3be9cb1551291fdcb03bfb98bc5
  Stored in directory: /root/.cache/pip/wheels/33/8b/99/cf0d27191876637cd3639a560f93aa982d7855ce826c94348b
  Building wheel for unidic (setup.py) ... done
  Created wheel for unidic: filename=unidic-1.0.3-py3-none-any.whl size=5506 sha256=8977d77c05cd5ec120e9f048171cc7fe4804c6191cb8c3dc209a1eae232c6aa8
  Stored in directory: /root/.cache/pip/wheels/23/30/0b/128289fb595ef4117d2976ffdbef5069ef83be813e88caa0a6
  Building wheel for unidic-lite (setup.py) ... done
  Created wheel for unidic-lite: filename=unidic_lite-1.0.8-py3-none-any.whl size=47658836 sha256=05af03831b7babce0692a95538db2a5616e083838fd8a98d6336cdf8e6f9a009
  Stored in directory: /root/.cache/pip/wheels/de/69/b1/112140b599f2b13f609d485a99e357ba68df194d2079c5b1a2
Successfully built ipadic unidic unidic-lite
Installing collected packages: tokenizers, sacremoses, unidic-lite, unidic, transformers, ipadic, fugashi
Successfully installed fugashi-1.1.1 ipadic-1.0.0 sacremoses-0.0.45 tokenizers-0.10.3 transformers-4.4.2 unidic-1.0.3 unidic-lite-1.0.8

from transformers import pipeline

# テキスト分類のパイプラインを準備
nlp = pipeline('sentiment-analysis')

# テキスト
text = 'This movie was very interesting.'

# 推論
print(nlp(text))

Downloading: 100%
629/629 [00:00<00:00, 10.1kB/s]
Downloading: 100%
268M/268M [00:06<00:00, 45.4MB/s]
Downloading: 100%
232k/232k [00:00<00:00, 266kB/s]
Downloading: 100%
48.0/48.0 [00:00<00:00, 850B/s]
[{'label': 'POSITIVE', 'score': 0.9997621178627014}]

from transformers import pipeline

# テキスト分類のパイプラインを準備
nlp = pipeline('sentiment-analysis')

# テキスト
text = 'This movie was very silly.'

# 推論
print(nlp(text))

[{'label': 'NEGATIVE', 'score': 0.9983341693878174}]

// テキストインデックスを作成
> db.test.createIndex({name:"text"})
{
        "numIndexesBefore" : 1,
        "numIndexesAfter" : 2,
        "createdCollectionAutomatically" : false,
        "ok" : 1
}

// インデックスの確認
> db.test.getIndexes()
[
        {
                "v" : 2,
                "key" : {
                        "_id" : 1
                },
                "name" : "_id_"
        },
        {
                "v" : 2,
                "key" : {
                        "_fts" : "text",
                        "_ftsx" : 1
                },
                "name" : "name_text",
                "weights" : {
                        "name" : 1
                },
                "default_language" : "english",
                "language_override" : "language",
                "textIndexVersion" : 3
        }
]

// Dateフィールドを持つドキュメント作成
> db.ttltest.insert({time : ISODate("2021-06-09T10:20:05Z") } )
WriteResult({ "nInserted" : 1 })

// 作成したドキュメント確認
> db.ttltest.find()
{ "_id" : ObjectId("614bb5956e9eb9350501ee38"), "time" : ISODate("2021-06-09T10:20:05Z") }

// TTLインデックス作成
> db.ttltest.createIndex({time:1}, {expireAfterSeconds:60})
{
        "numIndexesBefore" : 1,
        "numIndexesAfter" : 2,
        "createdCollectionAutomatically" : false,
        "ok" : 1
}

// 作成したTTLインデックスの確認
> db.ttltest.getIndexes()
[
        {
                "v" : 2,
                "key" : {
                        "_id" : 1
                },
                "name" : "_id_"
        },
        {
                "v" : 2,
                "key" : {
                        "time" : 1
                },
                "name" : "time_1",
                "expireAfterSeconds" : 60
        }
]

// 60秒後にデータが自動削除されることを確認
> db.ttltest.find()

> db.group.reIndex()
{
        "nIndexesWas" : 1,
        "nIndexes" : 1,
        "indexes" : [
                {
                        "v" : 2,
                        "key" : {
                                "_id" : 1
                        },
                        "name" : "_id_"
                }
        ],
        "ok" : 1
}

> db.position.getIndexes()
[
        {
                "v" : 2,
                "key" : {
                        "_id" : 1
                },
                "name" : "_id_"
        },
        {
                "v" : 2,
                "key" : {
                        "name" : 1
                },
                "name" : "name_1"
        }
]

> db.position.dropIndexes("name_1")
{ "nIndexesWas" : 2, "ok" : 1 }

> db.position.getIndexes()
[ { "v" : 2, "key" : { "_id" : 1 }, "name" : "_id_" } ]