df = pd.concat([tweet,test])
df.shape

example = "New competition launched :https://www.kaggle.com/c/nlp-getting-started"

def remove_URL(text):
    url = re.compile(r'https?://\S+|www\.\S+')
    return url.sub(r'', text)

remove_URL(example)
df['text'] = df['text'].apply(lambda x : remove_URL(x))

example = """<div>
<h1>Real or Fake</h1>
<p>Kaggle </p>
<a href="https://www.kaggle.com/c/nlp-getting-started">getting started</a>
</div>"""

def remove_html(text):
    html = re.compile(r'<.*?>')
    return html.sub(r'',text)
print(remove_html(example))

df['text'] = df['text'].apply(lambda x : remove_html(x))

# Reference : https://gist.github.com/slowkow/7a7f61f495e3dbb7e3d767f97bd7304b
def remove_emoji(text):
    emoji_pattern = re.compile("["
                           u"\U0001F600-\U0001F64F"  # emoticons
                           u"\U0001F300-\U0001F5FF"  # symbols & pictographs
                           u"\U0001F680-\U0001F6FF"  # transport & map symbols
                           u"\U0001F1E0-\U0001F1FF"  # flags (iOS)
                           u"\U00002702-\U000027B0"
                           u"\U000024C2-\U0001F251"
                           "]+", flags=re.UNICODE)
    return emoji_pattern.sub(r'', text)

remove_emoji("Omg another Earthquake 😔😔")

df['text'] = df['text'].apply(lambda x: remove_emoji(x))

def remove_punct(text):
    table = str.maketrans('','', string.punctuation)
    return text.translate(table)

example = "I am a #king"
print(remove_punct(example))

df['text'] = df['text'].apply(lambda x : remove_punct(x))

!pip install pyspellchecker

from spellchecker import SpellChecker

spell = SpellChecker()
def correct_spellings(text):
    corrected_text = []
    misspelled_words = spell.unknown(text.split())
    for word in text.split():
        if word in misspelled_words:
            corrected_text.append(spell.correction(word))
        else:
            corrected_text.append(word)
    return " ".join(corrected_text)
        
text = "corect me plese"
correct_spellings(text)

def create_corpus(target):
    corpus=[]
    
    for x in tweet[tweet['target']==target]['text'].str.split():
        for i in x:
            corpus.append(i)
    return corpus

corpus = create_corpus(0)

dic=defaultdict(int)
for word in corpus:
    if word in stop:
        dic[word] += 1

top = sorted(dic.items(), key=lambda x:x[1], reverse=True)[:10] 

x,y=zip(*top)
plt.bar(x,y)

corpus=create_corpus(1)

dic=defaultdict(int)
for word in corpus:
    if word in stop:
        dic[word]+=1

top=sorted(dic.items(), key=lambda x:x[1],reverse=True)[:10] 

x,y=zip(*top)
plt.bar(x,y)

plt.figure(figsize=(10,5))
corpus = create_corpus(1)

dic = defaultdict(int)
import string
special = string.punctuation
for i in (corpus):
    if i in special:
        dic[i] += 1
        
x,y = zip(*dic.items())
plt.bar(x,y)

plt.figure(figsize=(10,5))
corpus = create_corpus(0)

dic = defaultdict(int)
import string
special = string.punctuation
for i in (corpus):
    if i in special:
        dic[i] += 1
        
x, y = zip(*dic.items())
plt.bar(x,y,color='green')

counter = Counter(corpus)
most = counter.most_common()
x=[]
y=[]
for word,count in most[:40]:
    if (word not in stop) :
        x.append(word)
        y.append(count)

sns.barplot(x=y, y=x)

def get_top_tweet_bigrams(corpus, n=None):
    vec = CountVectorizer(ngram_range=(2, 2)).fit(corpus)
    bag_of_words = vec.transform(corpus)
    sum_words = bag_of_words.sum(axis=0) 
    words_freq = [(word, sum_words[0, idx]) for word, idx in vec.vocabulary_.items()]
    words_freq =sorted(words_freq, key = lambda x: x[1], reverse=True)
    return words_freq[:n]

plt.figure(figsize=(10,5))
top_tweet_bigrams=get_top_tweet_bigrams(tweet['text'])[:10]
x,y=map(list,zip(*top_tweet_bigrams))
sns.barplot(x=y,y=x)

fig,(ax1,ax2)=plt.subplots(1,2,figsize=(10,5))
tweet_len = tweet[tweet['target']==1]['text'].str.len()
ax1.hist(tweet_len,color='red')
ax1.set_title('disaster tweets')
tweet_len=tweet[tweet['target']==0]['text'].str.len()
ax2.hist(tweet_len,color='green')
ax2.set_title('Not disaster tweets')
fig.suptitle('Characters in tweets')
plt.show()

fig,(ax1,ax2)=plt.subplots(1,2,figsize=(10,5))
tweet_len=tweet[tweet['target']==1]['text'].str.split().map(lambda x: len(x))
ax1.hist(tweet_len,color='red')
ax1.set_title('disaster tweets')
tweet_len=tweet[tweet['target']==0]['text'].str.split().map(lambda x: len(x))
ax2.hist(tweet_len,color='green')
ax2.set_title('Not disaster tweets')
fig.suptitle('Words in a tweet')
plt.show()

fig,(ax1,ax2)=plt.subplots(1,2,figsize=(10,5))
word=tweet[tweet['target']==1]['text'].str.split().apply(lambda x : [len(i) for i in x])
sns.distplot(word.map(lambda x: np.mean(x)),ax=ax1,color='red')
ax1.set_title('disaster')
word=tweet[tweet['target']==0]['text'].str.split().apply(lambda x : [len(i) for i in x])
sns.distplot(word.map(lambda x: np.mean(x)),ax=ax2,color='green')
ax2.set_title('Not disaster')
fig.suptitle('Average word length in each tweet')

import os
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
import numpy as np
from nltk.corpus import stopwords
from nltk.util import ngrams
from sklearn.feature_extraction.text import CountVectorizer
from collections import defaultdict
from collections import  Counter
plt.style.use('ggplot')
stop=set(stopwords.words('english'))
import re
from nltk.tokenize import word_tokenize
import gensim
import string
from keras.preprocessing.text import Tokenizer
from keras.preprocessing.sequence import pad_sequences
from tqdm import tqdm
from keras.models import Sequential
from keras.layers import Embedding,LSTM,Dense,SpatialDropout1D
from keras.initializers import Constant
from sklearn.model_selection import train_test_split
from keras.optimizers import Adam

tweet= pd.read_csv('../input/nlp-getting-started/train.csv')
test=pd.read_csv('../input/nlp-getting-started/test.csv')
tweet.head(10)

test.head(10)

x=tweet.target.value_counts()
sns.barplot(x.index,x)
plt.gca().set_ylabel('samples')

test_data = pd.read_csv('./test.csv', dtype=np.float32)
test_data = test_data.to_numpy() / 255.0
test_data = test_data.reshape(-1, 28, 28, 1)  # reshapeの一つのサイズが決まっているとき、もう一方を-1とすると、-1には元の形状から推測された値が入る。

test_tensor = torch.from_numpy(test_data).permute(0, 3, 1, 2)  # numpy ndarrayからPytorch tensorに変換し、次元の入れ替え(permute)
test_tensor = test_tensor.repeat(1, 3, 1, 1)  # データセットの繰り返し

images = test_tensor.to(device)
outputs = model(images)
_, predictions = torch.max(outputs, 1)  # 配列の最大値の要素を返す

predictions = predictions.cpu()
submission = pd.DataFrame({'ImageId': np.arange(1, (predictions.size(0) + 1)), 'Label': predictions})
submission.to_csv("submission.csv", index = False)
print("# 終了 #")

# ライブラリをインポート
import  numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os
from tqdm import tqdm
import seaborn as sns
from sklearn.model_selection import train_test_split
import warnings
warnings.filterwarnings("ignore")

import torch
import torch.nn as nn
import torchvision
from torch.utils.data import DataLoader, Dataset
from torchvision import transforms

# CSVファイルを読み込み、正解ラベルのデータを分ける
data = pd.read_csv('./train.csv', dtype = np.float32)
labels = data.pop('label').astype('int64')

# 0～255のピクセルデータを0～1に変換
data = data.to_numpy() / 255.0 # converting to numpy and normalizing between 0 and 1
labels = labels.to_numpy()

data = data.reshape(-1, 28, 28, 1)
labels = labels.reshape(-1,1)
print(labels.shape)

# 訓練データと評価データに分ける
x_train, x_val, y_train, y_val = train_test_split(data, labels, test_size=0.2)
print(f'x_train.shape: {x_train.shape}, x_val.shape: {x_val.shape}')

# カスタムデータセットの定義
class MNISTDataset(Dataset):
        
    def __init__(self, images, labels, transform = None):
        """Method to initilaize variables.""" 
        self.images = images
        self.labels = labels
        self.transform = transform

    def __getitem__(self, index):
        label = self.labels[index]
        image = self.images[index]
        
        if self.transform is not None:
            image = self.transform(image)
        image = image.repeat(3, 1, 1)
        return image, label

    def __len__(self):
        return len(self.images)

# データを0～1のテンソルに変換
train_set = MNISTDataset(x_train, y_train, transform=transforms.Compose([transforms.ToTensor()]))
val_set = MNISTDataset(x_val, y_val, transform=transforms.Compose([transforms.ToTensor()]))
all_data = MNISTDataset(data, labels, transform=transforms.Compose([transforms.ToTensor()]))

train_loader = DataLoader(train_set, batch_size=32)
val_loader = DataLoader(val_set, batch_size=32)
all_data_loader = DataLoader(all_data, batch_size=32)

# GPUが使えればGPUを使用
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

learning_rate = 0.001
num_classes = 10
num_epochs = 10
# TorchvisionからResNet-18（畳み込みニューラル ネットワーク）モデルをダウンロード
model = torchvision.models.resnet18(pretrained=True)  # Kaggleノートブックではここでダウンロードエラー
num_ftrs = model.fc.in_features
# 全結合を行う3層構造のニューラルネットワークを生成
model.fc = nn.Linear(num_ftrs, num_classes)

model.to(device)

criterion = nn.CrossEntropyLoss()

# 全てのパラメータが最適化されることを観察
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate, momentum=0.9)

# ７エポックごとにLRを0.1ずつ減らす
# LR range test：初期学習率を決める手段で、ある幅で学習率を徐々に増加させながらAccuracyないしLossを観察し決定する手法
exp_lr_scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)

# 損失関数と最適化アルゴリズムを生成
criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

# 学習率を更新する
def update_lr(optimizer, lr):
    for param_group in optimizer.param_groups:
        param_group['lr'] = lr

# 学習する
total_step = len(train_loader)
curr_lr = learning_rate
for epoch in range(num_epochs):
    for i, (images, labels) in enumerate(train_loader):
        images = images.to(device)
        labels = labels.to(device)

        # Forward pass（順伝搬：初期の入力を層ごとに処理して出力に向けて送ること）
        outputs = model(images)
        loss = criterion(outputs, labels.flatten())

        # Backward and optimize（逆伝播と最適化を行う）
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        if (i + 1) % 300 == 0:
            print(f'Epoch: {epoch + 1}/{num_epochs}, Loss: {loss.item()}')

# 評価する
model.eval()
with torch.no_grad():
    correct = 0
    total = 0
    for images, labels in val_loader:
        images = images.to(device)
        labels = labels.to(device)

        outputs = model(images)
        _, predicted = torch.max(outputs, 1)
        total += labels.size(0)
        correct += (predicted == labels.flatten()).sum()

    print(f'Test acc: {100 * correct / total}')

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os
from tqdm import tqdm
from sklearn.model_selection import train_test_split
import seaborn as sns
import warnings
warnings.filterwarnings("ignore")

import torch
import torch.nn as nn
from torch.autograd import Variable
import torchvision

data = pd.read_csv('../input/digit-recognizer/train.csv', dtype = np.float32)
labels = data.pop('label').astype('int32')

data.head() # let's see first five rows

labels.head() # after converting labels to `int32`

data = data.to_numpy() / 255.0 # converting to numpy and normalizing between 0 and 1
labels = labels.to_numpy()

x_train, x_val, y_train, y_val = train_test_split(data, labels, test_size=0.2)
print(f'x_train.shape: {x_train.shape}, x_val.shape: {x_val.shape}')

plt.figure(figsize=(12, 10))
for i in range(16):
    plt.subplot(4, 4,i+1)
    plt.xticks([])
    plt.yticks([])
    plt.imshow(x_train[i].reshape(28,28), cmap=plt.cm.binary)
    plt.xlabel(y_train[i])
plt.show()

x_train_tensor = torch.from_numpy(x_train)
y_train_tensor = torch.from_numpy(y_train).type(torch.LongTensor)

x_val_tensor = torch.from_numpy(x_val)
y_val_tensor = torch.from_numpy(y_val).type(torch.LongTensor)

print(f'x_train_tensor.dtype: {x_train_tensor.dtype}, y_train_tensor.dtype: {y_train_tensor.dtype}')

train_data = torch.utils.data.TensorDataset(x_train_tensor, y_train_tensor)
val_data = torch.utils.data.TensorDataset(x_val_tensor, y_val_tensor)

train_loader = torch.utils.data.DataLoader(dataset=train_data, batch_size=32, shuffle=True)
val_loader = torch.utils.data.DataLoader(dataset=val_data, batch_size=1, shuffle=False)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

learning_rate = 0.001
num_epochs = 10
num_classes = 10

# Convolutional neural network (two convolutional layers)
class ConvNet(nn.Module):
    def __init__(self, num_classes=10):
        super(ConvNet, self).__init__()
        self.layer1 = nn.Sequential(
            nn.Conv2d(1, 16, kernel_size=5, stride=1, padding=2),
            nn.BatchNorm2d(16),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2))

        self.layer2 = nn.Sequential(
            nn.Conv2d(16, 32, kernel_size=5, stride=1, padding=2),
            nn.BatchNorm2d(32),
            nn.ReLU(),
            nn.MaxPool2d(kernel_size=2, stride=2))

        self.fc = nn.Linear(7 * 7 * 32, num_classes)

    def forward(self, x):
        out = self.layer1(x)
        out = self.layer2(out)
        out = out.reshape(out.size(0), -1)
        out = self.fc(out)
        return out

model = ConvNet(num_classes).to(device)         # CNNモデルの生成

criterion = nn.CrossEntropyLoss()               # 交差エントロピーの生成
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)   # SGD Optimizer（確率的勾配降下法）の生成

iter_num = 0
loss_list = []
iteration_list = []
accuracy_list = []

# Train the model（）
total_step = len(train_loader)
for epoch in range(num_epochs):
    for i, (images, labels) in enumerate(train_loader):  # （訓練データごとにループ）
        images = images.view(32, 1, 28, 28).to(device)
        labels = labels.to(device)

        # Forward pass（順伝搬：初期の入力を層ごとに処理して出力に向けて送ること）
        outputs = model(images)
        loss = criterion(outputs, labels)

        # Backward and optimize（最適化、確率的勾配降下法）
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()

        iter_num += 1

        if iter_num % 50 == 0:
            # Calculate Accuracy（正解率の算出）
            correct = 0
            total = 0
            # Iterate through val dataset（評価データごとにループ）
            for images, labels in val_loader:
                test = images.view(1, 1, 28, 28).to(device)
                labels = labels.to(device)
                # Forward propagation（順伝搬：初期の入力を層ごとに処理して出力に向けて送ること）
                outputs = model(test)
                
                # Get predictions from the maximum value（予測値の取得）
                predicted = torch.max(outputs.data, 1)[1]
                
                # Total number of labels（正解ラベルの総数）
                total += len(labels)
                
                correct += (predicted == labels).sum()
            
            accuracy = 100 * correct / float(total)

            # store loss and iteration（損失・正解率を保持）
            loss_list.append(loss.data)
            iteration_list.append(iter_num)
            accuracy_list.append(accuracy)
        if iter_num % 500 == 0:
            # Print Loss（損失と正解率を表示）
            print('Iteration: {},  Loss: {:.4f},  Accuracy: {:.4f} %'.format(iter_num, loss.data, accuracy))

# Evaluation（評価）
correct = 0
total = 0
# Iterate through val dataset（データごとにループ）
for images, labels in val_loader:
    test = images.view(1, 1, 28, 28).to(device)
    labels = labels.to(device)
    # Forward propagation（順伝搬：初期の入力を層ごとに処理して出力に向けて送ること）
    outputs = model(test)

    # Get predictions from the maximum value（予測値の取得）
    predicted = torch.max(outputs.data, 1)[1]

    # Total number of labels（正解ラベルの総数）
    total += len(labels)

    correct += (predicted == labels).sum()

accuracy = 100 * correct / float(total)
print(accuracy)

# Visualize loss
plt.plot(iteration_list,loss_list)
plt.xlabel("Num. of Iters.")
plt.ylabel("Loss")
plt.title("Loss vs Num. of Iters.")
plt.show()

# Visualize Accuracy
plt.plot(iteration_list,accuracy_list)
plt.xlabel("Num. of Iters.")
plt.ylabel("Accuracy")
plt.title("Accuracyvs Num. of Iters.")
plt.show()

x_train, x_val, y_train, y_val = train_test_split(data, labels, test_size=0.2)
print('Train shape:', x_train.shape, 'Val shape:', x_val.shape)

x_train_tensor = torch.from_numpy(x_train)
y_train_tensor = torch.from_numpy(y_train).type(torch.LongTensor)

x_val_tensor = torch.from_numpy(x_val)
y_val_tensor = torch.from_numpy(y_val).type(torch.LongTensor)

train_dataset = TensorDataset(x_train_tensor, y_train_tensor)
val_dataset = TensorDataset(x_val_tensor, y_val_tensor)

train_loader = DataLoader(train_dataset, batch_size=32, shuffle=True)
val_loader = DataLoader(val_dataset, batch_size=32, shuffle=False)

device = torch.device('cpu' if torch.cuda.is_available() else 'cpu')

input_size = 28 * 28    # 入力層のサイズ
output_size = 10        # 出力層のサイズ
hidden_size = 100       # 隠れ層（中間層）のサイズ
learning_rate = 0.001   # 学習率
num_epochs = 10         # エポック数「一つの訓練データを何回繰り返して学習させるか」

class LogisticRegression(nn.Module):
    def __init__(self, input_size, output_size):
        super(LogisticRegression, self).__init__()
        # Linear model
        self.linear = nn.Linear(input_size, output_size)

    def forward(self, x):
        out = self.linear(x)
        return out

model = LogisticRegression(input_size, output_size).to(device)

# Cross Entropy Loss（交差エントロピー）
criterion = nn.CrossEntropyLoss()

# SGD Optimizer（確率的勾配降下法）
optimizer = torch.optim.SGD(model.parameters(), lr=learning_rate)

loss_list = []
iteration_list = []
accuracy_list = []
iter_num = 0
for epoch in range(num_epochs):
    for i, (images, labels) in enumerate(train_loader):
        # input and label（入力データと正解ラベルを設定）
        train = images.view(-1, 28*28)
        labels = labels

        # Forward pass（順伝搬：初期の入力を層ごとに処理して出力に向けて送ること）
        outputs = model(train)

        # Loss calculate（損失計算）
        loss = criterion(outputs, labels)

        # Backward and optimize（最適化、確率的勾配降下法）
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        iter_num += 1
        
        if (i+1) % 50 == 0:
            correct = 0
            total = 0
            for images, labels in val_loader:
                # Forward pass（順伝搬：初期の入力を層ごとに処理して出力に向けて送ること）
                images = images.view(-1, 28*28)
                outputs = model(images)

                # Predictions（予測）
                predicted = torch.max(outputs, 1)[1]

                # Total number of samples（サンプル数）
                total += labels.size(0)

                # Total correct predictions（正解数）
                correct += (predicted == labels).sum()

            accuracy = 100 * (correct/total)
            loss_list.append(loss.data)
            iteration_list.append(iter_num)
            accuracy_list.append(accuracy)
        if (iter_num+1) % 1000 == 0:
            # Print Loss（損失と正解率を表示）
            print('Iteration: {} Loss: {:0.4f} Val Accuracy: {:0.4f}%'.format(iter_num+1, loss.data, accuracy))

# Visualize loss（損失のグラフ化）
plt.plot(iteration_list,loss_list)
plt.xlabel("Num. of Iters.")
plt.ylabel("Loss")
plt.title("Logistic Regression: Loss vs Num. of Iters.")
plt.show()

# Visualize accuracy（正解率のグラフ化）
plt.plot(iteration_list,accuracy_list)
plt.xlabel("Num. of Iters.")
plt.ylabel("Accuracy")
plt.title("Logistic Regression: Accuracy vs Num. of Iters.")
plt.show()

import  numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os
from tqdm import tqdm
import seaborn as sns
from sklearn.model_selection import train_test_split
import warnings
warnings.filterwarnings("ignore")

import torch
import torch.nn as nn
import torchvision
from torch.utils.data import TensorDataset, DataLoader, Dataset

train_csv = pd.read_csv('../input/digit-recognizer/train.csv', dtype = np.float32)
test_csv = pd.read_csv('../input/digit-recognizer/test.csv', dtype = np.float32)
train_csv.head(5)

labels = train_csv.pop('label')
labels = labels.to_numpy() # convert to numpy array

data = train_csv.to_numpy() / 255.0 # normalization
print('Data shape:', data.shape, 'Labels shape:', labels.shape)

sns.countplot(labels)
plt.title('Class Distribution')

plt.figure(figsize=(12, 10))
for i in range(20):
    plt.subplot(5, 4, i+1)
    plt.grid(False)
    plt.xticks([])
    plt.yticks([])
    plt.imshow(data[i].reshape(28,28)) 

import numpy as np
import pandas as pd

import os
import re
import warnings
print(os.listdir("../input"))

#import io
#import requests

# url="https://github.com/thisisjasonjafari/my-datascientise-handcode/raw/master/005-datavisualization/titanic.csv"
# s=requests.get(url).content
#c=pd.read_csv(io.StringIO(s.decode('utf-8')))
# 上記処理でダウンロードできなかったため手動ダウンロードしファイル読み込み
c=pd.read_csv('../input/githubtitanic/titanic.csv')

test_data_with_labels = c
test_data = pd.read_csv('../input/titanic/test.csv')

test_data_with_labels.head()

test_data.head()

# ワーニングが表示されないように
warnings.filterwarnings('ignore')

# 名前からダブルクォーテーションを除外してるだけ
for i, name in enumerate(test_data_with_labels['name']):
    if '"' in name:
        test_data_with_labels['name'][i] = re.sub('"', '', name)
        
# 名前からダブルクォーテーションを除外してるだけ
for i, name in enumerate(test_data['Name']):
    if '"' in name:
        test_data['Name'][i] = re.sub('"', '', name)

survived = []

for name in test_data['Name']:
    # values[-1]は同姓同名の場合、最後にヒットしたほうの生存結果を取得している。
    survived.append(int(test_data_with_labels.loc[test_data_with_labels['name'] == name]['survived'].values[-1]))

submission = pd.read_csv('../input/titanic/gender_submission.csv')
submission['Survived'] = survived
submission.to_csv('submission.csv', index=False)