modeling_sagvit.py

import torch
from torch import nn
from transformers import AutoConfig, PretrainedConfig, AutoModel, PreTrainedModel
from transformers.models.auto import AutoConfig, CONFIG_MAPPING, MODEL_MAPPING
from transformers.utils import logging
from transformers.modeling_utils import ModelOutput
from huggingface_hub import PyTorchModelHubMixin

from torch_geometric.data import Batch
from model_components import EfficientNetV2FeatureExtractor, GATGNN, TransformerEncoder, MLPBlock
from graph_construction import build_graph_from_patches, build_graph_data_from_patches


###############################################################################
# SAG-ViT Model:
# This class combines:
# 1) CNN backbone to produce high-fidelity feature maps (Section 3.1),
# 2) Graph construction and GAT to refine local patch embeddings (Section 3.2 and 3.3),
# 3) A Transformer encoder to capture global relationships (Section 3.3),
# 4) A final MLP classifier.
###############################################################################


# Custom model registration
class SAGViTConfig(PretrainedConfig):
    model_type = "sagvit"

    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        self.d_model = kwargs.get("d_model", 64)
        self.dim_feedforward = kwargs.get("dim_feedforward", 64)
        self.gcn_hidden = kwargs.get("gcn_hidden", 128)
        self.gcn_out = kwargs.get("gcn_out", 64)
        self.hidden_mlp_features = kwargs.get("hidden_mlp_features", 64)
        self.in_channels = kwargs.get("in_channels", 2560)
        self.nhead = kwargs.get("nhead", 4)
        self.num_classes = kwargs.get("num_classes", 10)
        self.num_layers = kwargs.get("num_layers", 2)
        self.patch_size = kwargs.get("patch_size", (4, 4))


class SAGViTClassifier(PreTrainedModel):
    """
    SAG-ViT: Scale-Aware Graph Attention Vision Transformer

    This model integrates the following steps:
    - Extract multi-scale features from images using a CNN backbone (EfficientNetv2 here).
    - Partition the feature map into patches and build a graph where each node is a patch.
    - Use a Graph Attention Network (GAT) to refine patch embeddings based on local spatial relationships.
    - Utilize a Transformer encoder to model long-range dependencies and integrate multi-scale information.
    - Finally, classify the resulting representation into desired classes.

    Inputs:
    - x (Tensor): Input images (B, 3, H, W)

    Outputs:
    - out (Tensor): Classification logits (B, num_classes)
    """

    config_class = SAGViTConfig
    def __init__(self, config):
        super().__init__(config)

        self.patch_size = config.patch_size
        self.num_classes = config.num_classes

        # CNN feature extractor (frozen pre-trained EfficientNetv2)
        self.cnn = EfficientNetV2FeatureExtractor()

        # Graph Attention Network to process patch embeddings
        self.gcn = GATGNN(
            in_channels=config.in_channels,
            hidden_channels=config.gcn_hidden,
            out_channels=config.gcn_out,
        )

        # Learnable positional embedding for Transformer input
        self.positional_embedding = nn.Parameter(torch.randn(1, 1, config.d_model))
        # Extra embedding token (similar to class token) to summarize global info
        self.extra_embedding = nn.Parameter(torch.randn(1, config.d_model))

        # Transformer encoder to capture long-range global dependencies
        self.transformer_encoder = TransformerEncoder(
            d_model=config.d_model, 
            nhead=config.nhead, 
            num_layers=config.num_layers, 
            dim_feedforward=config.dim_feedforward,
        )

        # MLP classification head
        self.mlp = MLPBlock(config.d_model, config.hidden_mlp_features, config.num_classes)

    def forward(self, x, **kwargs):
        # Step 1: High-fidelity feature extraction from CNN
        feature_map = self.cnn(x)

        # Step 2: Build graphs from patches
        G_global_batch, patches = build_graph_from_patches(feature_map, self.patch_size)

        # Step 3: Convert to PyG Data format and batch
        data_list = build_graph_data_from_patches(G_global_batch, patches)
        device = x.device
        batch = Batch.from_data_list(data_list).to(device)

        # Step 4: GAT stage
        x_gcn = self.gcn(batch)

        # Step 5: Reshape GCN output back to (B, N, D)
        # The number of patches per image is determined by patch size and feature map dimensions.
        B = x.size(0)
        D = x_gcn.size(-1)
        # N is automatically inferred
        # Thus x_gcn is (B, D) now. We need a sequence dimension for the Transformer.
        # Let's treat each image-level embedding as one "patch token" plus an extra token:
        patch_embeddings = x_gcn.unsqueeze(1)  # (B, 1, D)

        # Add positional embedding
        patch_embeddings = patch_embeddings + self.positional_embedding  # (B, 1, D)

        # Add an extra learnable embedding (like a CLS token)
        patch_embeddings = torch.cat([patch_embeddings, self.extra_embedding.unsqueeze(0).expand(B, -1, -1)], dim=1)  # (B, 2, D)

        # Step 6: Transformer encoder
        x_trans = self.transformer_encoder(patch_embeddings)

        # Step 7: Global pooling (here we just take the mean)
        x_pooled = x_trans.mean(dim=1)  # (B, D)

        # Classification
        logits = self.mlp(x_pooled)
        return ModelOutput(logits=logits)

# Register custom model and config
CONFIG_MAPPING.register("sagvit", SAGViTConfig)
MODEL_MAPPING.register(SAGViTConfig, SAGViTClassifier)