#LLM #NLP #ML #AI #PYTHON #PYTORCH

https://yangx.top/DataScienceM

import torch
import torch.nn as nn

class BasicRNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(BasicRNN, self).__init__()
        self.rnn = nn.RNN(input_size, hidden_size, batch_first=True)
        self.fc = nn.Linear(hidden_size, output_size)

    def forward(self, x):
        out, _ = self.rnn(x)  # out: [batch, seq_len, hidden]
        out = self.fc(out[:, -1, :])  # Take the output from last time step
        return out

nn.RNN(input_size, hidden_size, bidirectional=True)

nn.RNN(input_size, hidden_size, num_layers=2)

import torch.nn as nn

class SentimentRNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super(SentimentRNN, self).__init__()
        self.rnn = nn.RNN(input_size, hidden_size, num_layers=1, batch_first=True)
        self.fc = nn.Linear(hidden_size, output_size)

    def forward(self, x):
        out, _ = self.rnn(x)
        final_out = out[:, -1, :]  # Get the last time-step output
        return self.fc(final_out)

nn.LSTM(input_size, hidden_size, num_layers=2, bidirectional=True)

torch.nn.utils.clip_grad_norm_(model.parameters(), max_norm=1.0)

packed_input = nn.utils.rnn.pack_padded_sequence(input, lengths, batch_first=True)

from sklearn.feature_extraction.text import TfidfVectorizer

texts = ["I love data science.", "Data science is fun."]
vectorizer = TfidfVectorizer(stop_words='english')
X = vectorizer.fit_transform(texts)

class VanillaRNN(nn.Module):
    def __init__(self, input_size, hidden_size, output_size):
        super().__init__()
        self.hidden_size = hidden_size
        self.rnn = nn.RNN(input_size, hidden_size, batch_first=True)
        self.fc = nn.Linear(hidden_size, output_size)
        
    def forward(self, x, hidden=None):
        # x shape: (batch, seq_len, input_size)
        out, hidden = self.rnn(x, hidden)
        # Only use last output for classification
        out = self.fc(out[:, -1, :])  
        return out

# Usage
rnn = VanillaRNN(input_size=10, hidden_size=20, output_size=5)
x = torch.randn(3, 15, 10)  # (batch=3, seq_len=15, input_size=10)
output = rnn(x)

class LSTMModel(nn.Module):
    def __init__(self, input_size, hidden_size, num_layers, output_size):
        super().__init__()
        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, 
                           batch_first=True, dropout=0.2 if num_layers>1 else 0)
        self.fc = nn.Linear(hidden_size, output_size)
        
    def forward(self, x):
        # Initialize hidden state and cell state
        h0 = torch.zeros(self.lstm.num_layers, x.size(0), 
                        self.lstm.hidden_size).to(x.device)
        c0 = torch.zeros_like(h0)
        
        out, (hn, cn) = self.lstm(x, (h0, c0))
        out = self.fc(out[:, -1, :])
        return out

# Bidirectional LSTM example
bidir_lstm = nn.LSTM(input_size=10, hidden_size=20, num_layers=2,
                    bidirectional=True, batch_first=True)

# Learning rate scheduler for transformers
def lr_schedule(step, d_model=512, warmup_steps=4000):
    arg1 = step ** -0.5
    arg2 = step * (warmup_steps ** -1.5)
    return (d_model ** -0.5) * min(step ** -0.5, step * warmup_steps ** -1.5)

# Character-level LSTM starter
class CharLSTM(nn.Module):
    def __init__(self, vocab_size, hidden_size, n_layers):
        super().__init__()
        self.embed = nn.Embedding(vocab_size, hidden_size)
        self.lstm = nn.LSTM(hidden_size, hidden_size, n_layers, batch_first=True)
        self.fc = nn.Linear(hidden_size, vocab_size)
        
    def forward(self, x, hidden=None):
        x = self.embed(x)
        out, hidden = self.lstm(x, hidden)
        return self.fc(out), hidden