!pip install 'kaggle-environments>=0.1.6'

from kaggle_environments import evaluate, make, utils

env = make("connectx", debug=True)
env.render()

# This agent random chooses a non-empty column.
def my_agent(observation, configuration):
    from random import choice
    return choice([c for c in range(configuration.columns) if observation.board[c] == 0])

env.reset()
# Play as the first agent against default "random" agent.
env.run([my_agent, "random"])
env.render(mode="ipython", width=500, height=450)

# Play as first position against random agent.
trainer = env.train([None, "random"])

observation = trainer.reset()

while not env.done:
    my_action = my_agent(observation, env.configuration)
    print("My Action", my_action)
    observation, reward, done, info = trainer.step(my_action)
    # env.render(mode="ipython", width=100, height=90, header=False, controls=False)
env.render()

def mean_reward(rewards):
    return sum(r[0] for r in rewards) / float(len(rewards))

# Run multiple episodes to estimate its performance.
print("My Agent vs Random Agent:", mean_reward(evaluate("connectx", [my_agent, "random"], num_episodes=10)))
print("My Agent vs Negamax Agent:", mean_reward(evaluate("connectx", [my_agent, "negamax"], num_episodes=10)))

# "None" represents which agent you'll manually play as (first or second player).
env.play([None, "negamax"], width=500, height=450)

import inspect
import os

def write_agent_to_file(function, file):
    with open(file, "a" if os.path.exists(file) else "w") as f:
        f.write(inspect.getsource(function))
        print(function, "written to", file)

write_agent_to_file(my_agent, "submission.py")

# Note: Stdout replacement is a temporary workaround.
import sys
out = sys.stdout
submission = utils.read_file("/kaggle/working/submission.py")
agent = utils.get_last_callable(submission)
sys.stdout = out

env = make("connectx", debug=True)
env.run([agent, agent])
print("Success!" if env.state[0].status == env.state[1].status == "DONE" else "Failed...")

model = Sequential()

embedding=Embedding(num_words, 100, embeddings_initializer=Constant(embedding_matrix),
                   input_length=MAX_LEN, trainable=False)

model.add(embedding)
model.add(SpatialDropout1D(0.2))
model.add(LSTM(64, dropout=0.2, recurrent_dropout=0.2))
model.add(Dense(1, activation='sigmoid'))

optimzer = Adam(learning_rate=1e-5)

model.compile(loss='binary_crossentropy',optimizer=optimzer,metrics=['accuracy'])

model.summary()

train=tweet_pad[:tweet.shape[0]]
test=tweet_pad[tweet.shape[0]:]

X_train,X_test,y_train,y_test = train_test_split(train,tweet['target'].values,test_size=0.15)
print('Shape of train',X_train.shape)
print("Shape of Validation ",X_test.shape)

history=model.fit(X_train,y_train,batch_size=4,epochs=15,validation_data=(X_test,y_test),verbose=2)

sample_sub=pd.read_csv('../input/nlp-getting-started/sample_submission.csv')

y_pre = model.predict(test)
y_pre = np.round(y_pre).astype(int).reshape(3263)
sub = pd.DataFrame({'id':sample_sub['id'].values.tolist(), 'target':y_pre})
sub.to_csv('submission.csv',index=False)

def create_corpus(df):
    corpus=[]
    for tweet in tqdm(df['text']):
        words=[word.lower() for word in word_tokenize(tweet) if((word.isalpha()==1) & (word not in stop))]
        corpus.append(words)
    return corpus

corpus = create_corpus(df)

embedding_dict={}
with open('../input/glove-global-vectors-for-word-representation/glove.6B.100d.txt','r') as f:
    for line in f:
        values = line.split()
        word = values[0]
        vectors = np.asarray(values[1:],'float32')
        embedding_dict[word]  =vectors
f.close()

MAX_LEN = 50
tokenizer_obj = Tokenizer()
tokenizer_obj.fit_on_texts(corpus)
sequences = tokenizer_obj.texts_to_sequences(corpus)

tweet_pad = pad_sequences(sequences, maxlen=MAX_LEN, truncating='post', padding='post')

word_index = tokenizer_obj.word_index
print('Number of unique words:',len(word_index))

num_words = len(word_index) + 1
embedding_matrix = np.zeros((num_words, 100))

for word,i in tqdm(word_index.items()):
    if i > num_words:
        continue
    
    emb_vec = embedding_dict.get(word)
    if emb_vec is not None:
        embedding_matrix[i] = emb_vec

df = pd.concat([tweet,test])
df.shape

example = "New competition launched :https://www.kaggle.com/c/nlp-getting-started"

def remove_URL(text):
    url = re.compile(r'https?://\S+|www\.\S+')
    return url.sub(r'', text)

remove_URL(example)
df['text'] = df['text'].apply(lambda x : remove_URL(x))

example = """<div>
<h1>Real or Fake</h1>
<p>Kaggle </p>
<a href="https://www.kaggle.com/c/nlp-getting-started">getting started</a>
</div>"""

def remove_html(text):
    html = re.compile(r'<.*?>')
    return html.sub(r'',text)
print(remove_html(example))

df['text'] = df['text'].apply(lambda x : remove_html(x))

# Reference : https://gist.github.com/slowkow/7a7f61f495e3dbb7e3d767f97bd7304b
def remove_emoji(text):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    return emoji_pattern.sub(r'', text)

remove_emoji("Omg another Earthquake 😔😔")

df['text'] = df['text'].apply(lambda x: remove_emoji(x))

def remove_punct(text):
    table = str.maketrans('','', string.punctuation)
    return text.translate(table)

example = "I am a #king"
print(remove_punct(example))

df['text'] = df['text'].apply(lambda x : remove_punct(x))

!pip install pyspellchecker

from spellchecker import SpellChecker

spell = SpellChecker()
def correct_spellings(text):
    corrected_text = []
    misspelled_words = spell.unknown(text.split())
    for word in text.split():
        if word in misspelled_words:
            corrected_text.append(spell.correction(word))
        else:
            corrected_text.append(word)
    return " ".join(corrected_text)
        
text = "corect me plese"
correct_spellings(text)

def create_corpus(target):
    corpus=[]
    
    for x in tweet[tweet['target']==target]['text'].str.split():
        for i in x:
            corpus.append(i)
    return corpus

corpus = create_corpus(0)

dic=defaultdict(int)
for word in corpus:
    if word in stop:
        dic[word] += 1

top = sorted(dic.items(), key=lambda x:x[1], reverse=True)[:10] 

x,y=zip(*top)
plt.bar(x,y)

corpus=create_corpus(1)

dic=defaultdict(int)
for word in corpus:
    if word in stop:
        dic[word]+=1

top=sorted(dic.items(), key=lambda x:x[1],reverse=True)[:10] 

x,y=zip(*top)
plt.bar(x,y)

plt.figure(figsize=(10,5))
corpus = create_corpus(1)

dic = defaultdict(int)
import string
special = string.punctuation
for i in (corpus):
    if i in special:
        dic[i] += 1
        
x,y = zip(*dic.items())
plt.bar(x,y)

plt.figure(figsize=(10,5))
corpus = create_corpus(0)

dic = defaultdict(int)
import string
special = string.punctuation
for i in (corpus):
    if i in special:
        dic[i] += 1
        
x, y = zip(*dic.items())
plt.bar(x,y,color='green')

counter = Counter(corpus)
most = counter.most_common()
x=[]
y=[]
for word,count in most[:40]:
    if (word not in stop) :
        x.append(word)
        y.append(count)

sns.barplot(x=y, y=x)

def get_top_tweet_bigrams(corpus, n=None):
    vec = CountVectorizer(ngram_range=(2, 2)).fit(corpus)
    bag_of_words = vec.transform(corpus)
    sum_words = bag_of_words.sum(axis=0) 
    words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]
    words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
    return words_freq[:n]

plt.figure(figsize=(10,5))
top_tweet_bigrams=get_top_tweet_bigrams(tweet['text'])[:10]
x,y=map(list,zip(*top_tweet_bigrams))
sns.barplot(x=y,y=x)

fig,(ax1,ax2)=plt.subplots(1,2,figsize=(10,5))
tweet_len = tweet[tweet['target']==1]['text'].str.len()
ax1.hist(tweet_len,color='red')
ax1.set_title('disaster tweets')
tweet_len=tweet[tweet['target']==0]['text'].str.len()
ax2.hist(tweet_len,color='green')
ax2.set_title('Not disaster tweets')
fig.suptitle('Characters in tweets')
plt.show()

fig,(ax1,ax2)=plt.subplots(1,2,figsize=(10,5))
tweet_len=tweet[tweet['target']==1]['text'].str.split().map(lambda x: len(x))
ax1.hist(tweet_len,color='red')
ax1.set_title('disaster tweets')
tweet_len=tweet[tweet['target']==0]['text'].str.split().map(lambda x: len(x))
ax2.hist(tweet_len,color='green')
ax2.set_title('Not disaster tweets')
fig.suptitle('Words in a tweet')
plt.show()

fig,(ax1,ax2)=plt.subplots(1,2,figsize=(10,5))
word=tweet[tweet['target']==1]['text'].str.split().apply(lambda x : [len(i) for i in x])
sns.distplot(word.map(lambda x: np.mean(x)),ax=ax1,color='red')
ax1.set_title('disaster')
word=tweet[tweet['target']==0]['text'].str.split().apply(lambda x : [len(i) for i in x])
sns.distplot(word.map(lambda x: np.mean(x)),ax=ax2,color='green')
ax2.set_title('Not disaster')
fig.suptitle('Average word length in each tweet')

import os
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from nltk.corpus import stopwords
from nltk.util import ngrams
from sklearn.feature_extraction.text import CountVectorizer
from collections import defaultdict
from collections import  Counter
plt.style.use('ggplot')
stop=set(stopwords.words('english'))
import re
from nltk.tokenize import word_tokenize
import gensim
import string
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from tqdm import tqdm
from keras.models import Sequential
from keras.layers import Embedding,LSTM,Dense,SpatialDropout1D
from keras.initializers import Constant
from sklearn.model_selection import train_test_split
from keras.optimizers import Adam

tweet= pd.read_csv('../input/nlp-getting-started/train.csv')
test=pd.read_csv('../input/nlp-getting-started/test.csv')
tweet.head(10)

test.head(10)

x=tweet.target.value_counts()
sns.barplot(x.index,x)
plt.gca().set_ylabel('samples')

test_data = pd.read_csv('./test.csv', dtype=np.float32)
test_data = test_data.to_numpy() / 255.0
test_data = test_data.reshape(-1, 28, 28, 1)  # reshapeの一つのサイズが決まっているとき、もう一方を-1とすると、-1には元の形状から推測された値が入る。

test_tensor = torch.from_numpy(test_data).permute(0, 3, 1, 2)  # numpy ndarrayからPytorch tensorに変換し、次元の入れ替え(permute)
test_tensor = test_tensor.repeat(1, 3, 1, 1)  # データセットの繰り返し

images = test_tensor.to(device)
outputs = model(images)
_, predictions = torch.max(outputs, 1)  # 配列の最大値の要素を返す

predictions = predictions.cpu()
submission = pd.DataFrame({'ImageId': np.arange(1, (predictions.size(0) + 1)), 'Label': predictions})
submission.to_csv("submission.csv", index = False)
print("# 終了 #")

# ライブラリをインポート
import  numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os
from tqdm import tqdm
import seaborn as sns
from sklearn.model_selection import train_test_split
import warnings
warnings.filterwarnings("ignore")

import torch
import torch.nn as nn
import torchvision
from torch.utils.data import DataLoader, Dataset
from torchvision import transforms

# CSVファイルを読み込み、正解ラベルのデータを分ける
data = pd.read_csv('./train.csv', dtype = np.float32)
labels = data.pop('label').astype('int64')

# 0～255のピクセルデータを0～1に変換
data = data.to_numpy() / 255.0 # converting to numpy and normalizing between 0 and 1
labels = labels.to_numpy()

data = data.reshape(-1, 28, 28, 1)
labels = labels.reshape(-1,1)
print(labels.shape)

# 訓練データと評価データに分ける
x_train, x_val, y_train, y_val = train_test_split(data, labels, test_size=0.2)
print(f'x_train.shape: {x_train.shape}, x_val.shape: {x_val.shape}')

# カスタムデータセットの定義
class MNISTDataset(Dataset):
        
    def __init__(self, images, labels, transform = None):
        """Method to initilaize variables.""" 
        self.images = images
        self.labels = labels
        self.transform = transform

    def __getitem__(self, index):
        label = self.labels[index]
        image = self.images[index]
        
        if self.transform is not None:
            image = self.transform(image)
        image = image.repeat(3, 1, 1)
        return image, label

    def __len__(self):
        return len(self.images)

# データを0～1のテンソルに変換
train_set = MNISTDataset(x_train, y_train, transform=transforms.Compose([transforms.ToTensor()]))
val_set = MNISTDataset(x_val, y_val, transform=transforms.Compose([transforms.ToTensor()]))
all_data = MNISTDataset(data, labels, transform=transforms.Compose([transforms.ToTensor()]))

train_loader = DataLoader(train_set, batch_size=32)
val_loader = DataLoader(val_set, batch_size=32)
all_data_loader = DataLoader(all_data, batch_size=32)

# GPUが使えればGPUを使用
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

learning_rate = 0.001
num_classes = 10
num_epochs = 10
# TorchvisionからResNet-18（畳み込みニューラル ネットワーク）モデルをダウンロード
model = torchvision.models.resnet18(pretrained=True)  # Kaggleノートブックではここでダウンロードエラー
num_ftrs = model.fc.in_features
# 全結合を行う3層構造のニューラルネットワークを生成
model.fc = nn.Linear(num_ftrs, num_classes)

model.to(device)

criterion = nn.CrossEntropyLoss()

# 全てのパラメータが最適化されることを観察
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9)

# ７エポックごとにLRを0.1ずつ減らす
# LR range test：初期学習率を決める手段で、ある幅で学習率を徐々に増加させながらAccuracyないしLossを観察し決定する手法
exp_lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)

# 損失関数と最適化アルゴリズムを生成
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

# 学習率を更新する
def update_lr(optimizer, lr):
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

# 学習する
total_step = len(train_loader)
curr_lr = learning_rate
for epoch in range(num_epochs):
    for i, (images, labels) in enumerate(train_loader):
        images = images.to(device)
        labels = labels.to(device)

        # Forward pass（順伝搬：初期の入力を層ごとに処理して出力に向けて送ること）
        outputs = model(images)
        loss = criterion(outputs, labels.flatten())

        # Backward and optimize（逆伝播と最適化を行う）
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if (i + 1) % 300 == 0:
            print(f'Epoch: {epoch + 1}/{num_epochs}, Loss: {loss.item()}')

# 評価する
model.eval()
with torch.no_grad():
    correct = 0
    total = 0
    for images, labels in val_loader:
        images = images.to(device)
        labels = labels.to(device)

        outputs = model(images)
        _, predicted = torch.max(outputs, 1)
        total += labels.size(0)
        correct += (predicted == labels.flatten()).sum()

    print(f'Test acc: {100 * correct / total}')

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os
from tqdm import tqdm
from sklearn.model_selection import train_test_split
import seaborn as sns
import warnings
warnings.filterwarnings("ignore")

import torch
import torch.nn as nn
from torch.autograd import Variable
import torchvision

data = pd.read_csv('../input/digit-recognizer/train.csv', dtype = np.float32)
labels = data.pop('label').astype('int32')

data.head() # let's see first five rows

labels.head() # after converting labels to `int32`

data = data.to_numpy() / 255.0 # converting to numpy and normalizing between 0 and 1
labels = labels.to_numpy()

x_train, x_val, y_train, y_val = train_test_split(data, labels, test_size=0.2)
print(f'x_train.shape: {x_train.shape}, x_val.shape: {x_val.shape}')

plt.figure(figsize=(12, 10))
for i in range(16):
    plt.subplot(4, 4,i+1)
    plt.xticks([])
    plt.yticks([])
    plt.imshow(x_train[i].reshape(28,28), cmap=plt.cm.binary)
    plt.xlabel(y_train[i])
plt.show()

x_train_tensor = torch.from_numpy(x_train)
y_train_tensor = torch.from_numpy(y_train).type(torch.LongTensor)

x_val_tensor = torch.from_numpy(x_val)
y_val_tensor = torch.from_numpy(y_val).type(torch.LongTensor)

print(f'x_train_tensor.dtype: {x_train_tensor.dtype}, y_train_tensor.dtype: {y_train_tensor.dtype}')

train_data = torch.utils.data.TensorDataset(x_train_tensor, y_train_tensor)
val_data = torch.utils.data.TensorDataset(x_val_tensor, y_val_tensor)

train_loader = torch.utils.data.DataLoader(dataset=train_data, batch_size=32, shuffle=True)
val_loader = torch.utils.data.DataLoader(dataset=val_data, batch_size=1, shuffle=False)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

learning_rate = 0.001
num_epochs = 10
num_classes = 10

# Convolutional neural network (two convolutional layers)
class ConvNet(nn.Module):
    def __init__(self, num_classes=10):
        super(ConvNet, self).__init__()
        self.layer1 = nn.Sequential(
            nn.Conv2d(1, 16, kernel_size=5, stride=1, padding=2),
            nn.BatchNorm2d(16),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2))

        self.layer2 = nn.Sequential(
            nn.Conv2d(16, 32, kernel_size=5, stride=1, padding=2),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2))

        self.fc = nn.Linear(7 * 7 * 32, num_classes)

    def forward(self, x):
        out = self.layer1(x)
        out = self.layer2(out)
        out = out.reshape(out.size(0), -1)
        out = self.fc(out)
        return out

model = ConvNet(num_classes).to(device)         # CNNモデルの生成

criterion = nn.CrossEntropyLoss()               # 交差エントロピーの生成
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)   # SGD Optimizer（確率的勾配降下法）の生成

iter_num = 0
loss_list = []
iteration_list = []
accuracy_list = []

# Train the model（）
total_step = len(train_loader)
for epoch in range(num_epochs):
    for i, (images, labels) in enumerate(train_loader):  # （訓練データごとにループ）
        images = images.view(32, 1, 28, 28).to(device)
        labels = labels.to(device)

        # Forward pass（順伝搬：初期の入力を層ごとに処理して出力に向けて送ること）
        outputs = model(images)
        loss = criterion(outputs, labels)

        # Backward and optimize（最適化、確率的勾配降下法）
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        iter_num += 1

        if iter_num % 50 == 0:
            # Calculate Accuracy（正解率の算出）
            correct = 0
            total = 0
            # Iterate through val dataset（評価データごとにループ）
            for images, labels in val_loader:
                test = images.view(1, 1, 28, 28).to(device)
                labels = labels.to(device)
                # Forward propagation（順伝搬：初期の入力を層ごとに処理して出力に向けて送ること）
                outputs = model(test)
                
                # Get predictions from the maximum value（予測値の取得）
                predicted = torch.max(outputs.data, 1)[1]
                
                # Total number of labels（正解ラベルの総数）
                total += len(labels)
                
                correct += (predicted == labels).sum()
            
            accuracy = 100 * correct / float(total)

            # store loss and iteration（損失・正解率を保持）
            loss_list.append(loss.data)
            iteration_list.append(iter_num)
            accuracy_list.append(accuracy)
        if iter_num % 500 == 0:
            # Print Loss（損失と正解率を表示）
            print('Iteration: {},  Loss: {:.4f},  Accuracy: {:.4f} %'.format(iter_num, loss.data, accuracy))

# Evaluation（評価）
correct = 0
total = 0
# Iterate through val dataset（データごとにループ）
for images, labels in val_loader:
    test = images.view(1, 1, 28, 28).to(device)
    labels = labels.to(device)
    # Forward propagation（順伝搬：初期の入力を層ごとに処理して出力に向けて送ること）
    outputs = model(test)

    # Get predictions from the maximum value（予測値の取得）
    predicted = torch.max(outputs.data, 1)[1]

    # Total number of labels（正解ラベルの総数）
    total += len(labels)

    correct += (predicted == labels).sum()

accuracy = 100 * correct / float(total)
print(accuracy)

# Visualize loss
plt.plot(iteration_list,loss_list)
plt.xlabel("Num. of Iters.")
plt.ylabel("Loss")
plt.title("Loss vs Num. of Iters.")
plt.show()

# Visualize Accuracy
plt.plot(iteration_list,accuracy_list)
plt.xlabel("Num. of Iters.")
plt.ylabel("Accuracy")
plt.title("Accuracyvs Num. of Iters.")
plt.show()