\documentclass[a4paper,10pt,twoside]{article}
\usepackage{informat}
\usepackage{epsfig}
\usepackage{enumitem} % 加载 enumitem 宏包
\usepackage{amsmath} % 加载 amsmath 宏包
\usepackage[ruled,vlined]{algorithm2e}
\usepackage{booktabs}
\usepackage{multirow}
\usepackage{amssymb} % 提供 \checkmark
\usepackage{xcolor} % 必须加载
\usepackage{diagbox}  % 用于绘制斜线分割单元格
\usepackage{url}
% \usepackage{hyperref}
\usepackage[colorlinks=true, linkcolor=black, urlcolor=black, citecolor=black, pdfborder={0 0 0}]{hyperref}
\usepackage[numbers,sort&compress]{natbib} % 连续的多文献序号进行压缩显示: sort&compress

%\usepackage{fontspec}
%\setmainfont{Times New Roman}

\begin{document}
%\setcounter{page}{555}
\title{A Visual Question Answering Debiasing Method Integrating Bias Detection and Forgotten Attention Mechanism}

\author{Qiuying Han$^1$, Shaohui Zhang$^{2,3}$, Peng Wang$^{2}$, Boyuan Li$^{3,*}$, and Qiwen Lu$^{4,*}$\\
      $^1$ School of Computer Science and Technology, Zhoukou Normal University, Zhoukou 466001, China\\
      $^2$ School of Artificial Intelligence, Zhoukou Normal University, Zhoukou 466001, China\\
      $^3$ School of Computer and Artificial Intelligence, Zhengzhou University, Zhengzhou 450001, China\\
      $^4$ School of Computer and Information Engineering, Henan University, Zhengzhou 450046, China\\
      E-mail: hanqy@zknu.edu.cn, zhangsh@zknu.edu.cn, wangp@zknu.edu.cn, ieboyuan@163.com, easonglll@he-nu.edu.cn\\
      $^*$Corresponding Author}

\titleodd{A Visual Question Answering Debiasing Method \ldots}  % 页眉短标题
% Title: A Visual Question Answering Debiasing Method Integrating Bias Detection and Forgotten Attention Mechanism

\authoreven{Q. Han et al.}

\keywords{Visual Question Answering, Language Bias, Multi-modal Reasoning, Attention Mechanism, Forgotten Attention Algorithm, Ensemble Models, Gradient Debiasing Approach}
\received{April 19, 2025}

\abstract{In Visual Question Answering (VQA), the model's ability to understand and reason across different modalities—language, visual, and multimodal—is crucial for accurate predictions. However, recent studies have identified a significant challenge of language bias in VQA models, where the model's reasoning often hinges on incorrect linguistic associations rather than genuine multimodal understanding. To address this issue, we propose a novel approach that integrates bias detection with a dynamic forgetting mechanism. Our method utilizes a bias detector to identify and score biases across linguistic, visual, and multimodal data, enabling the model to focus on unbiased information for more accurate predictions. Additionally, inspired by human reasoning, we introduce the Forgotten Attention Algorithm (FAA), which iteratively ``forgets'' irrelevant visual content, progressively concentrating attention on the image regions most relevant to the question. This combination of bias mitigation and attention focusing enhances the model's ability to make multimodal inferences, reducing bias and improving overall performance. Extensive experiments on the VQA-CP v2, VQA v2, VQA-VS, GQA-OOD, and VQA-CE datasets demonstrate the effectiveness of our approach, showing state-of-the-art performance in mitigating biases and excelling in complex multimodal scenarios.}
\abstractSi{}

% \keyword{Visual Question Answering; Language Bias; Multi-modal Reasoning; Attention Mechanism; Forgotten Attention Algorithm; Ensemble Models; Gradient Debiasing Approach} 


\maketitle

\section{Introduction}
In recent years, Visual Question Answering (VQA) has become one of the prominent tasks in the field of deep learning \cite{hudson2019learning}, achieving significant accomplishments in various applications, such as intelligent service systems \cite{Luo_Lin_Yao_Liu_Liu_Tang_2023, Wang_Pradhan_Gunasekaran_2022}. However, recent research has found that many existing VQA methods tend to rely on false associations between questions and answers, without sufficiently extracting accurate visual information from images to answer questions. For example, when answering questions ``What color?'', some VQA models are inclined to use the most common answers from training data of that type, like ``yellow'', rather than extracting genuine color information from images. Additionally, some studies \cite{XinzheHan2021GreedyGE,liu2022answer} have indicated deficiencies in the existing methods' understanding of images, resulting in answers generated by the model relying on image regions with low relevance to the questions. In other words, specific methods often provide correct answers based on incorrect image regions, which does not genuinely reflect the model's performance in the question-answering task. Consequently, the factors affecting the robustness of VQA models can be summarized into two primary aspects: inherent biases in the language distribution of training and testing datasets, and the improper shortcut biases caused by the inadequate utilization of visual information \cite{Liu_Fan_Zhou_Xu_2023}. However, biases in VQA are multifaceted, affecting not only language but also the improper use of visual information, resulting in models that often focus on irrelevant regions of the image. As shown in Fig. \ref{fig:intro}, when the object elements in the image are nearly the same, the model during training will generate the answer ``Standing'' based on the object and keyword (``What'' and ``doing''). However, during testing, for the same question, the model will generate the incorrect answer based on similar objects and keywords (the correct answer is ``Jumping'', but the model gives the wrong answer ``Standing'').
\begin{figure}
    \includegraphics[width=\linewidth]{image/fig1_new.pdf}
    \caption{An example of multimodal bias: the presence of visual features related to the dog and horse, along with the keyword ``what'' and ``doing'' in the question, misleads the model into predicting an incorrect answer. While the correct action for the second image is ``Jumping'', the model incorrectly predicts ``Standing'' based on similar objects and keywords.}
    \label{fig:intro}
\end{figure}

The state-of-the-art and noteworthy methods primarily revolve around data augmentation techniques and attention-based approaches. Data augmentation methods \cite{chen2020counterfactual} aim to enhance a model's understanding of critical features within the data by expanding the dataset with samples, such as counterfactual instances and additional annotations \cite{liang2020learning, gokhale2020mutant}, which help eliminate biases and enhance robustness \cite{agarwal2020towards,wen2021debiased} by obtaining more critical sample features and supplementary information. However, it is still of great interest and challenge to remove the language biases in VQA model without resorting to data augmentation \cite{YuleiNiu2021CounterfactualVA}. Regarding attention-based methods \cite{PeterAnderson2017BottomUpAT}, the majority currently integrate these into pre-trained models for efficient feature fusion \cite{Tan_Bansal_2019, Yu_Yu_Cui_Tao_Tian_2019,Lu_Yang_Batra_Parikh_2016,Lu_Li_Zhang_Wang_Wang_2022,PeterAnderson2017BottomUpAT}, with limited emphasis on fully utilizing visual information.

\begin{figure*}[t]
  \includegraphics[width=\linewidth]{image/introduction_new_attention.pdf}
  \caption{Due to the presence of biases, the influence of the size of prominent objects in the image on model reasoning leads to incorrect answers, while the image regions relevant to the answers often occupy a small portion. FAA achieves this by masking irrelevant regions in the image, allowing the model to focus on image details for inference.}
  \label{attention:fig2}
\end{figure*}
We introduce an Ensemble Bias Gradient Debiasing Approach (EBGDA), which incorporates ensemble learning to dynamically identify and score different types of biases in the data. The method applies overfitting techniques to certain biases to ensure that the main model focuses on unbiased and relevant information during prediction. Our bias detector, tailored to the multimodal nature of VQA, assesses the degree of bias in both linguistic and visual domains, helping the model to adapt its learning process accordingly.

Additionally, inspired by the idea of iterative human reasoning, where we progressively focus on relevant areas of an image based on the question, we propose a novel approach called the Forgotten Attention Algorithm (FAA). This algorithm progressively ``forgets'' irrelevant image regions after each iteration, allowing the model to concentrate on the most relevant visual information for answering the question. In Fig. \ref{attention:fig2}, it is evident that prominent objects (\textit{i.e.}, the bench) often dominate the model's attention, causing it to overlook the finer image area that is relevant to the question (\textit{i.e.}, the people). This observation poses a new challenge: how to focus on the right image area that is the most relevant to the question. To address this problem, we are inspired by the process of answering questions of human beings, where people always gradually reduce the focus area in the image with the aid of question information until the final related area is retained.  This iterative forgetting process helps the model mitigate the impact of biased associations by focusing on the crucial regions of the image that relate to the question, improving its reasoning capacity.

To validate the effectiveness of our approach, we conducted extensive experiments on the VQA-CP v2, VQA v2, VQA-VS, GQA and VQA-VE. The results show that our method, combining FAA and EBGDA, significantly outperforms existing models, without relying on additional annotations or data augmentation. Our approach effectively mitigates language and visual biases, enhancing the robustness of VQA models across various challenging scenarios.
%
The main contributions of this study are summarized as follows:

\begin{enumerate}[label=(\arabic*)]
    \item Ensemble Bias Gradient Debiasing Approach: We propose a unified multi-bias debiasing approach which leverages ensemble learning to dynamically identify and evaluate the impact of linguistic, visual, and multimodal biases. The method selectively overfits to harmful biases, enabling the main model to focus more effectively on unbiased and relevant information during prediction.
    
    \item Cognitive-Inspired Attention Mechanism: We propose a novel Forgotten Attention Algorithm (FAA) inspired by human cognitive processes, which progressively refines visual focus by iteratively ``forgetting'' irrelevant image regions, thereby enhancing alignment between question-guided attention and critical visual content.

    \item Bias Mitigation via Deliberate Forgetting: The FAA addresses language bias in VQA by systematically suppressing spurious question-answer correlations through multi-round attention refinement, forcing the model to rely on genuine multi-modal reasoning rather than linguistic shortcuts.

    \item Iterative Attention Refinement for Precision: Unlike conventional static attention mechanisms, our FAA dynamically narrows the model’s focus across successive iterations, ensuring precise localization of question-relevant visual features while discarding distracting information.
    
    \item Extensive Experimental Validation: On VQA-CP v2, our enhancements in leveraging visual information led to optimal performance. Notably, without additional annotations, our approach attained a 21.32\% improvement compared to the UpDn baseline model.

    % Code is available at:\url{https://github.com/EASONGLLL/FAA-VQA}.
\end{enumerate}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Related works}
\subsection{Visual Question Answering}
The Visual Question Answering (VQA) task requires models to provide accurate answers to natural language questions based on visual content. Since its introduction, significant progress has been made in dataset development and multimodal fusion techniques. Benchmark datasets such as VQA v2 \cite{antol2015vqa}, GQA \cite{DrewAHudson2019GQAAN}, and CLEVR \cite{JustinJohnson2016CLEVRAD} have driven advancements by providing large-scale annotated samples with balanced linguistic distributions. For more complex reasoning, OK-VQA \cite{KennethMarino2019OKVQAAV} focuses on outside-knowledge-based questions, while VideoQA \cite{KeweiTu2013JointVA} extends the task to dynamic video understanding.

Current state-of-the-art approaches primarily adopt single-stream or dual-stream architectures \cite{WeiYang2019EndtoEndOQ,ZhiguoWang2019MultipassageBA,GautierIzacard2021LeveragingPR,PranavRajpurkar2018KnowWY,YenChunChen2020UNITERUI}. Single-stream models (\textit{e.g.}, ViLBERT \cite{ZhiguoWang2019MultipassageBA}, LXMERT \cite{GautierIzacard2021LeveragingPR}) employ unified transformers to jointly encode image-text pairs, enabling deep cross-modal interaction. In contrast, dual-stream methods (\textit{e.g.}, MCAN \cite{PranavRajpurkar2018KnowWY}, UNITER \cite{YenChunChen2020UNITERUI}) process visual and linguistic features separately before fusion, often achieving higher precision through modular design. These models benefit from large-scale pretraining on datasets like Conceptual Captions \cite{Sharma_Ding_Goodman_Soricut_2018} or Visual Genome \cite{Krishna_Zhu_Groth_Johnson_Hata_Kravitz_Chen_Kalantidis_Li_Shamma_etal}, which enhances their ability to align visual and textual representations.

However, despite their strong performance, these methods still face two critical challenges:
\begin{enumerate}[label=(\arabic*)]
    \item Language Bias: Models tend to exploit statistical priors in training data (\textit{e.g.}, favoring frequent answers like ``yellow'' for color questions) rather than grounding responses in visual evidence \cite{han2021greedy}.

     \item Weak Visual Grounding: Attention mechanisms often focus on salient but irrelevant objects (\textit{e.g.}, a ``bench'' dominating attention when the question concerns ``people''), leading to incorrect reasoning paths \cite{wen2021debiased}.
\end{enumerate}

Recent work attempts to address these issues through debias strategies (\textit{e.g.}, adversarial learning \cite{jing2020overcoming}, counterfactual samples \cite{chen2020counterfactual}) and enhanced attention mechanisms (\textit{e.g.}, compositional attention \cite{Yu_Yu_Cui_Tao_Tian_2019}). Yet, most methods either rely on costly data augmentation or fail to fully leverage visual cues. Our work bridges this gap by introducing a human-inspired iterative attention refinement approach, eliminating language biases without external data.


\subsection{Mitigate Language Bias}
In recent research, researchers have proposed a range of debiasing methods to address language bias concerning existing defined bias issues. These methods include adversarial-based techniques \cite{ramakrishnan2018overcoming}, regularization approaches \cite{YuleiNiu2021CounterfactualVA,XinzheHan2021GreedyGE,Abbasnejad_Teney_Parvaneh_Shi_Hengel_2020,Cho_Kim_Ryu_Kweon_2023,Basu_Addepalli_Babu_2023}, and data augmentation strategies. Our approach focuses on addressing bias issues from the perspective of the visual modality.  % 原文

The pervasive issue of language bias in VQA systems has been extensively documented in recent literature \cite{YuleiNiu2021CounterfactualVA,XinzheHan2021GreedyGE}. These biases manifest when models exploit statistical regularities in question-answer pairings while neglecting genuine visual evidence, leading to compromised generalization capabilities, particularly on out-of-distribution datasets like VQA-CP \cite{chen2020counterfactual}. Existing methodologies for mitigating language bias can be categorized into three primary paradigms:
\begin{enumerate}[label=(\arabic*)]
\item Adversarial-based Techniques: 
Methods such as \cite{ramakrishnan2018overcoming} employ gradient reversal layers or auxiliary adversarial networks to explicitly disentangle language priors from visual reasoning pathways. While effective in theory, these approaches often face optimization challenges, as noted in \cite{grand2019adversarial}, where the adversarial training dynamics can inadvertently suppress semantically relevant linguistic patterns along with harmful biases.

\item Regularization Approaches: Recent advances in regularization-based debiasing employ specialized loss functions to mitigate language prior overfitting while preserving model generalizability. Representative approaches include: a question-only branch with entropy maximization to discourage shortcut learning \cite{XinzheHan2021GreedyGE}, attention divergence constraints to ensure visual grounding \cite{Cho_Kim_Ryu_Kweon_2023}, and a contrastive learning framework to distinguish between vision-relevant and bias-driven features \cite{Basu_Addepalli_Babu_2023}. These approaches demonstrate that structured regularization can effectively reduce linguistic bias without compromising model capacity, offering computationally efficient alternatives to adversarial training or data augmentation.

\item Data Augmentation Strategies: Techniques like \cite{chen2020counterfactual,wen2021debiased,HUAI2024PRL} generate counterfactual samples or synthetic QA pairs to break spurious correlations. The VQA-CP dataset \cite{chen2020counterfactual} represents a notable example of this approach through systematic answer distribution inversion. However, such methods may introduce new biases during synthetic data generation while incurring significant annotation costs\cite{HUAI2024PRL}.
\end{enumerate}





\subsection{Attention Mechanism}
In the context of Visual Question Answering (VQA), attention mechanisms are employed to integrate information from different modalities \cite{PeterAnderson2017BottomUpAT}, allowing models to focus on the most relevant regions between images and texts. Presently, attention-based methodologies include linear attention \cite{ZichaoYang2016StackedAN}, co-attention \cite{Lu_Yang_Batra_Parikh_2016}, detection attention \cite{PeterAnderson2017BottomUpAT}, and relational attention \cite{Wu_Liu_Wang_Dong_2018}. Consequently, in our approach, we explore the integration of attention mechanisms into debiasing methods in VQA, strengthening the model's retrieval capabilities between images and questions. Leveraging attention mechanisms enhances the role of visual information, ultimately aiding in debiasing strategies.
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Method}
We describe the architecture and algorithmic flow of FAA. As shown in Fig. \ref{attention fig3}, the left side illustrates the primary structure of the UpDn baseline model \cite{PeterAnderson2017BottomUpAT}, responsible for extracting visual-language features. On the right side, there are stacked $Attention\_Layers$ that iteratively mask irrelevant features and make answer predictions.

\begin{figure*}[htbp] % 图3
    \includegraphics[height=0.4\textwidth, width=1.0\textwidth]{image/FAA_method_New.pdf}
    \caption{Our proposed FAA follows the architecture of the UpDn baseline model, comprising the feature extraction stage and the attention layers. The attention layers aim to retrieve information from the encoded question and image features, facilitating a multi-round retrieval process. In each round of retrieval, an image mask matrix is constructed to mask out the information deemed irrelevant by the model during this round, retaining crucial information for subsequent reasoning.}
    \label{attention fig3}
\end{figure*}

\subsection{Visual Information Combination}
On the left side of Fig. \ref{attention fig3}, we utilize the UpDn encoding layer to extract features. For a given text, the UpDn leverages a standard GRU to encode each question, generating a question vector. Regarding the provided image, UpDn uses the detected visual features as input. The visual feature set is represented as $F = \{f_{1},..f_{i}..,f_{n}\}$, where $f_{i}$ denotes the feature of the $i$-th object in the image. In our method, we also incorporate factors such as spatial position. We re-encode all the outputs from Faster-RCNN \cite{Ren_He_Girshick_Sun_2017} into new visual features. The visual input $V$ is represented as Eq. \eqref{eq1}:
\begin{equation}
    \label{eq1}
    \setlength\abovedisplayskip{6pt}
    \setlength\belowdisplayskip{6pt}
    V = Visual\_Encoder(F, S, Cls, Ari)
\end{equation}
where $Visual\_Encoder$ represents the visual encoder responsible for re-encoding the four types of features into visual input. These four types of features are represented as visual feature vectors $F$, spatial features $S$, classification scores $Cls$, and attribute information $Ari$. During the initialization phase, this re-encoded visual data $V$ is introduced as the visual input for the VQA process.
\subsection{Bias Definition}
% 对于VQA任务，传统方法通常将其看作一个多分类任务来处理。模型通常接收给定的三元组数据$D = \{v_{i}, q_{i}, a_{i}\} _{i=1} ^{N}$作为输入，其中$i$-th  image $v_{i} \in \mathcal{V}$, question $q_{i} \in \mathcal{Q}$, answer $a_{i} \in \mathcal{A}$.其主要目的是为了训练一个映射用来根据输入数据在答案集$\mathcal{A}$上生成准确的答案分布。在VQA-VS中，作者基于三种模态设置了九种不同类型的偏见，分别是语言，视觉和多模态类型的偏见。在实际的工作中，我们以VS数据集为主要研究数据集，对于三种模态对应的偏见，我们延续之前工作的内容，使用单独模型对其建模，用来捕获模态中的偏见。同时，我们将视觉问答中的偏见现象看作是模态在推理过程中的缺失。具体而言，对于输入样本对$(V,Q)$，我们设置了三个偏见模型对样本对进行偏见特征建模。
For the VQA, traditional methods typically treat it as a multi-class classification problem. The model generally takes a given triplet dataset $D = \{v_{i}, q_{i}, a_{i}\} _{i=1} ^{N}$ as input, where the $i$-th image is $v_{i} \in \mathcal{V}$, the question is $q_{i} \in \mathcal{Q}$, and the answer is $a_{i} \in \mathcal{A}$. The main goal is to train a mapping that generates an accurate answer distribution over the answer set $\mathcal{A}$ based on the input data. In VQA-VS, the authors define nine different types of biases across three modalities, namely, linguistic, visual, and multimodal biases. In practice, we focus on the VS dataset as the primary research dataset, and for the biases corresponding to the three modalities, we follow the previous work by modeling each modality individually to capture biases within each modality. Furthermore, we treat bias in VQA as the absence of modality-specific reasoning during the inference process. Specifically, for the input sample pair $(V,Q)$, we establish three bias models to perform bias feature modeling on the sample pair.
\begin{align}
\label{eq: bias}
    & B_{q} = c_{q}(\sigma(q_{i}))\\
    & B_{v} = c_{v}(\sigma(v_{i}))\\
    & B_{m} = c_{m}(\sigma(m_{i}))
\end{align}

% 其中，$q_{i},v_{i},m_{i}$分别表示问题，图像和混合表示，$c_{*}$表示用来捕获语言，图像，多模态类型偏见的模型，$B_{*}$表示对应的模型捕获的偏见特征结果。
where $q_{i}$, $v_{i}$, and $m_{i}$ represent the question, image, and multimodal representation, respectively. The term $c_{*}, *\in \{q,v,m\}$ denotes the models used to capture biases in the language, visual, and multimodal types, while $B_{*}, *\in \{q,v,m\}$ refers to the bias feature outcomes captured by the corresponding models.
% 对于语言和视觉偏见，我们在方法中认为模型在预测时某种模态缺失，
% QT+KO KW+KO QT+KW+KO
\subsection{Bias Detection}
% 首先，我们认为在实际的预测中，模型需要具备能够识别不同类型的偏见，这样在面对不同类型的数据时也具有相当的泛化性；其次在视觉问答中，并非所有的偏见都对模型只有负面作用，例如以关键字或关键对象为主要类型的偏见类型，模型在预测时可能会需要一些额外现实知识，这些知识可以作为补充知识来进行准确预测，而非偏见误导模型进行预测。因此，本文设置了一种偏见检测器，不仅用来辅助模型进行偏见识别，还可以对识别到的偏见进行权重估计，以衡量该偏见对模型的影响作用。

% 具体来说，我们根据上一步所获得的偏见特征结果以及偏见定义，计算对应偏见特征的损失来确定模型当前遇到的偏见是何种偏见。
Firstly, we believe that in practical prediction tasks, a model must be able to identify different types of biases, which enables it to maintain strong generalization performance when handling diverse types of data. Additionally, in VQA, not all biases necessarily have a purely negative impact on the model. For instance, bias based on keywords or key objects may require the model to incorporate some additional real-world knowledge, which can act as supplementary information for making accurate predictions, rather than misleading the model. Therefore, in this paper, we propose a bias detector that not only assists the model in identifying biases but also estimates the weights of the detected biases to assess their influence on the model's performance.

Specifically, based on the bias feature results and definitions obtained in the previous step, we calculate the loss of the corresponding bias features to determine the type of bias the model is currently encountering.
\begin{align}
\label{eq:loss}
    Loss_{q} = \mathcal{L}(B_q, A)\\
    Loss_{v} = \mathcal{L}(B_v, A)\\
    Loss_{m} = \mathcal{L}(B_m, A)
\end{align}
% 其中，$A$表示数据集中的真实答案，$Loss_{*}$分别表示对应偏见特征结果计算的损失。在方法中，我们选择计算偏见特征结果与真实标签之间损失的原因在于，偏见特征对应的结果与真实结果之间计算的损失如果过大，则认为偏见模型得到的预测结果很差，也就是说对偏见建模的能力很差，在集成模型的方法中我们需要将偏见模型充分训练以实现过拟合的方法令整体模型对偏见具有相当的识别能力。
where $A$ represents the ground truth answers in the dataset, and $Loss_{*}$ denotes the loss calculated for the corresponding bias feature results. The reason for calculating the loss between the bias feature results and the ground truth labels in our method is that if the loss between the bias feature and the actual result is too large, it indicates that the predictions generated by the bias model are poor, meaning that the model has a weak ability to capture the bias. In the ensemble model approach, we aim to sufficiently train the bias model to achieve overfitting, so that the overall model develops a strong capacity for bias recognition.

In summary, in our method, we treat the modality with the largest $Loss_{*}$ as the one where the model exhibits bias, and we mark it accordingly.
\begin{equation}
    F = B_*  
\end{equation}
where $F$ represents the bias flag. $ B_* $ represents the corresponding modality bias in $ \max(Loss_{q}, Loss_{v}, Loss_{m}) $.

\subsection{Bias Score}
After identifying the biases present in the sample pair, we use the bias detector to score the bias features, with the scores reflecting the degree of harm the bias imposes on the model (\textit{i.e.}, whether the bias is beneficial or harmful). If the bias in the current sample aids the model’s performance, there is no need to overfit to it during training. However, if the bias harms the model, it is necessary to ensure the model thoroughly learns the features corresponding to that bias during training. Accordingly, beneficial biases receive higher scores, while harmful biases are assigned lower scores.
\begin{align}
    \label{eq:score}
    score = softmax(m_{c}(F,A))
\end{align}
% 其中，$F$表示上一步中获得的偏见标志，$m_{c}$表示线性层，将标记对应的偏见特征与标签进行线性拟合。最终经过softmax层计算得到偏见分数。
Here, $F$ represents the bias flag obtained in the previous step, and $m_{c}$ denotes the linear layer, which linearly fits the bias features corresponding to the flag with the labels. Finally, the bias score is computed through the softmax layer.

\subsection{Fitting Gradients}
% 根据得到的分数，我们在方法中延续了Boosting思想，使用不同的分类器来过拟合训练数据。而对于具有偏见的弱特征，则使用过拟合的方法来学习偏差分布，利用有偏模型梯度训练的基础模型，从而去除VQA中的语言偏差。
Based on the obtained scores, we follow the Boosting concept in our method by employing different classifiers to overfit the training data. For weak features associated with bias, we apply an overfitting approach to learn the bias distribution. By leveraging a biased model's gradient to train the base model, we aim to eliminate language bias in VQA tasks.

% 在训练中，基于集成模型的思想，方法的主要目标是通过拟合主模型，偏见模型与标签的之间的差距。因此对于本方法中的三种偏见模型，我们定义为$M = \{M_{v}, M_{q}, M_{m}\}$,
% 最终目的是将三个模型和主模型的预测集合与标签 $Y$ 进行拟合
In training, based on the ensemble model concept, the main goal of the method is to minimize the gap between the predictions of the main model, bias models, and the true labels. Therefore, the three bias models in this method are defined as $M = \{M_{v}, M_{q}, M_{m}\}$, representing the visual, question, and multimodal bias models, respectively. The ultimate objective is to fit the prediction set of these three models, along with the main model, to the ground truth labels $Y$.

\begin{equation}
    \mathrm{min}\mathcal{L}(\sigma(\sum_{i = v}^{v,q,m} M_{i} + f(v,q)), Y)
\end{equation}
% 其中$M$表示有偏模型，$f$表示神经网络。我们希望数据集中有偏见的部分只通过有偏模型进行拟合。在方法中，我们将使用梯度的反方向作为伪标签，指导模型对偏见训练数据过拟合\cite{}。我们在方法中区分了不同类型的偏见对模型的影响，结合上一步得到的偏见得分，在训练数据进行梯度下降的过程中对梯度的大小进行放缩。简单来说，训练数据中包含的有利偏见对应的分数较高，而在反向梯度传播时下降的程度就较低，模型需要利用这样的偏见来获取知识；对于那些数据中有害的偏见，需要在反向传播时放大其下降的程度
Here, $M$ represents the biased model, and $f$ denotes the neural network. Our goal is to ensure that the biased parts of the dataset are fit exclusively through the biased model. In our method, we use the opposite direction of the gradient as pseudo-labels to guide the model in overfitting the biased training data. We differentiate between the effects of different types of biases on the model, and based on the bias scores obtained in the previous step, we scale the magnitude of the gradient during the gradient descent process. Simply put, higher scores correspond to beneficial biases in the training data, resulting in less decrease during backpropagation, as the model should leverage such biases to acquire knowledge. Conversely, for harmful biases, the gradient descent is amplified during backpropagation to reduce the model's reliance on such biases.
\begin{align}
    & Y_{new} = -\nabla \mathcal{L}(f(v,q))\\
    & \mathrm{min}\mathcal{L}(\sigma(\sum_{i = v}^{v,q,m} M_{i} + f(v,q)), s * Y_{new})
\end{align}
% 而在测试阶段，我们只使用主模型进行预测，令主模型进行正常数据的预测。  
Here, $ s $ represents the bias score.
During the testing phase, we only use the main model for predictions, allowing the main model to handle the prediction of unbiased data under normal conditions.

\subsection{Attention Layers}
In the right side of Fig. \ref{attention fig3}, we have stacked $N$ layers of $Attention\_Layer$ to achieve visual information masking and retrieval. Specifically, the $Attention\_Layer$ module consists of three main components:

\begin{enumerate}[label=(\arabic*)]
    \item\label{first} \textbf{Initial Impression}. After obtaining visual and text features, the next step in our process is to employ the $Self\_Attention$ mechanism. This mechanism helps identify the most critical components within each modality, similar to how humans instinctively react when first encountering an image or text. We establish the model's initial assessment of the pivotal image regions and word vectors within the provided features. 
    As formally delineated in Algorithm \ref{alg:faa}, this process can be mathematically defined as follows:
            \begin{equation}
            \begin{aligned}
                \label{eq2}
                &att_{v} = Self(V)\\
                &att_{q} = Self(Q)\\
                &V^{1} = att_{v} * V\\
                &Q^{1} = att_{q} * Q
            \end{aligned}
            \end{equation}
    where $att_{v}$ and $att_{q}$ represent the initial attention. $V^{1}$ and $Q^{1}$ represent the features obtained after fusing the initial attention with the original data.


\begin{algorithm}[htbp]
    \caption{Forgetting Attention Algorithm}
    \small
    \label{alg:faa}
    \SetAlgoNlRelativeSize{-1}
    \SetAlgoNlRelativeSize{0}
    \SetAlgoNlRelativeSize{1}
    \SetAlgoNlRelativeSize{0}
    
    \SetKwInOut{Input}{Input}
    \SetKwInOut{Output}{Output}
    \SetKwProg{Fn}{Function}{:}{end}
    
    \Input{Representation of Object Detection Outputs:$\mathcal{F}, \mathcal{S}, Cls, Ari$;
          Text coded representation: $Q$;
          Number of layers of attention stack: $N$;
          Attention threshold: $\alpha$.}
    \Output{Predicted answer probability:$\mathcal{A}$.}
    \BlankLine
    Initialize:$V \leftarrow [\mathcal{F}, \mathcal{S}, Cls, Ari]$, $k \leftarrow 3$.\\
    \Fn{FAA($V$, $Q$)}{
        \While{$n \leq N$}{
            $att_{v}, att_{q} \leftarrow SelfAttention(V, Q)$\\
            $V^{1}, Q^{1} \leftarrow  att_{v} \odot V, att_{q} \odot Q$\\
            $V^{2}, Q^{2} \leftarrow CrossAttention(V^{1}, Q^{1})$\\
            $Att \leftarrow V^{2} \odot Q^{2}$\\
            \eIf{$Att \leq \alpha$}{
              $V_{mask} \leftarrow 1$\;
              }{
              $V_{mask} \leftarrow 0$\;
              }
            $V^{3} \leftarrow V_{mask} \oplus V^{2}$\\
            $V, Q \leftarrow V^{3}, Q^{2}$
        }
        $\mathcal{A} \leftarrow V^{3}, Q^{2}$
    }
    return $\mathcal{A}$
\end{algorithm}

    
    \item\label{cross} \textbf{Cross-Modal Retrieval}. With the obtained features $V^{1}$ and $Q^{1}$, we consider using the Cross Attention mechanism \cite{Tan_Bansal_2019} to explore information across modalities. This step is analogous to how humans associate objects with words. We perform cross-modal information retrieval separately in the image and text domains. This is defined as Eq. \eqref{eq4}:
            \begin{equation}
                \label{eq4}
                \begin{aligned}
                    &V^{2} = CrossAtt_{v \rightarrow q}(Q^1, V^1)\\
                    &Q^{2} = CrossAtt_{q \rightarrow v}(V^1, Q^1)
                \end{aligned}
            \end{equation}
where $V^{2}$ and $Q^{2}$ represent the feature outputs after conducting cross-modal retrieval for the image and text, respectively. $CrossAtt$ respectively represents the cross-modal information retrieval layer, with ``image''  and ``question''  as the primary modalities.

    \item\label{mask} \textbf{Masking Matrix}. After Cross-Modal Retrieval, we calculate the masking matrix for $V^{2}$ and $Q^{2}$. Initially, we employ the Top-Down attention mechanism \cite{PeterAnderson2017BottomUpAT} to obtain an attention weight matrix: $Att$, which is then compared to a predefined threshold $\alpha$ to determine the masking matrix. As depicted in Algorithm \ref{alg:faa}, this is defined as Eq. \eqref{eq5}:
            \begin{equation}
                \label{eq5}
                \begin{aligned}
                    & Att = V^{2} * Q^{2}\\    
                    & V_{mask} = Mask(Att \leq \alpha)\\
                    & V^3 = V_{mask} \oplus V^2
                \end{aligned}
            \end{equation}
where $V_{mask}$ represents the masking matrix, and $Mask()$ denotes the process in which $Att$ is compared to $\alpha$ in Algorithm \ref{alg:faa}. The value of $\alpha$ is determined by the mean of attention.
$V^{3}$ represents the features obtained by merging the masking matrix with visual features. $\oplus$ denotes the linear fusion of two types of features. 
\end{enumerate}

Specifically, in each attention layer, we establish a masking matrix based on the magnitude of attention weights, which masks regions in the image that contribute less to the answer. Through $N$ such attention layers, we allow the model to progressively identify precise regions with high relevance to the given question.


\subsection{Training Phase}
Our model is designed to effectively identify and mitigate various biases that arise in VQA tasks. The process begins with the integration of multimodal features, where visual information is extracted using object detection techniques such as Faster R-CNN.  These visual features, including object location, classification scores, and spatial attributes, are then combined with textual features derived from the question to form a comprehensive representation for each input pair.
%
A key component of this phase is the Bias Detection mechanism, as shown in Fig. \ref{attention fig4}.  Here, the model evaluates and scores biases in both the linguistic and visual domains.  The linguistic biases, such as word associations, are detected through a specialized language bias model, while visual biases are identified by analyzing the model's tendency to focus on irrelevant or dominant visual features.  By dynamically assessing these biases, the model adjusts its learning process to ensure that it prioritizes unbiased information, allowing it to better reason across both modalities.

\begin{figure*}[t]   % 图 4. 训练阶段图
    \centering
    \includegraphics[height=0.28\textwidth, width=\textwidth]{image/model_intro.pdf}
    \caption{Training Phase Diagram.}
    \label{attention fig4}
\end{figure*}


Furthermore, the FAA is employed to progressively filter out irrelevant image regions during training.  This iterative attention mechanism helps the model to focus on the most relevant parts of the image that contribute to answering the question, simulating the human-like reasoning of gradually reducing focus to the most pertinent visual areas.  The FAA works by masking less important image regions in each iteration, ensuring that the model refines its attention and concentrates on the critical visual details.

Through this training process, which integrates bias detection, bias scoring, and the FAA, the model becomes more robust and capable of handling complex, real-world VQA scenarios without relying on biased shortcuts.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Experimental analysis and discussion}
In this section, we will present the quantitative and qualitative analysis results from the experiments.
\subsection{Setup}
\textbf{Dataset and evaluation metric.} We validated our method on multiple VQA datasets: VQA v2, VQA-CP v2, VQA-VS, GQA, and VQA-CE. The VQA-CP v2 is employed to evaluate bias in VQA models across various scenarios, while GQA-OOD assesses the model's robustness on both ID and OOD data. VQA-CE and VQA-VS concentrate on multimodal biases, analyzing shortcut learning patterns to provide a more detailed evaluation of OOD performance. Standard evaluation on VQA v2 gauges the approach's generalizability. All test results are based on standard VQA evaluation metrics.

\textbf{Baseline.} We mainly used the UpDn\cite{anderson2018bottom} model as the baseline model.

% \subsection{Comparisons with State-of-the-Arts}
\subsection{Comparisons with SOTAs}
We compared our method with other bias reduction methods, as shown in Table \ref{all result}. First, we introduce the method names and their corresponding baseline models, followed by the experimental results on VQA-CP v2 and VQA v2, respectively.

In Table \ref{all result}, our method achieves state-of-the-art performance on VQA-CP v2 (62.78\%). Moreover, compared to other methods using the same baseline models, our approach demonstrates superior performance across all three categories, particularly achieving the best results in the Num. and Other categories.

On VQA v2, we also attain state-of-the-art performance (65.47\%), ranking in the top three across all specific categories.

We further compared our method to approaches that utilize supplementary data. As seen, our method remains competitive against methods like CSS and D-VQA, which are trained on balanced datasets, even though such comparisons are considered unfair in \cite{niu2021counterfactual}.

\begin{table*}[htbp] % 表1
    % \caption{本模型在VQA-CP v2 数据集和VQA v2 验证集下与其他模型的实验结果对比。}
    \caption{The results of VQA-CP v2 test set and VQA v2 validation set are presented in the following table. Each column illustrates the {\textbf{Best}} performances of each method, excluding data augmentation techniques. Our method has been compared with state-of-the-art methods on both datasets.}
    \label{all result}
    \setlength{\extrarowheight}{2pt}
    \resizebox{\textwidth}{!}{
    \begin{tabular}{@{}lccccccccc@{}}
    \toprule
    % {Data set} & {} & \multicolumn{4}{c}{VQA-CP v2 test} & \multicolumn{4}{c}{VQA v2 val} \\
    % \cmidrule[0.5pt](rl){3-6}
    % \cmidrule[0.5pt](rl){7-10}
    % {Method} & {Base} & {All} & {Y/N} & {Num.} & {Other} & {All} & {Y/N} & {Num.} & {Other} \\ 
    \multirow{2}{*}{\diagbox[width=35mm, height=10mm]{Method}{Data Sets}}  & {} & \multicolumn{4}{c}{VQA-CP v2 test} & \multicolumn{4}{c}{VQA v2 val} \\
    \cmidrule[0.5pt](rl){3-6}
    \cmidrule[0.5pt](rl){7-10}
     & {Base} & {All} & {Y/N} & {Num.} & {Other} & {All} & {Y/N} & {Num.} & {Other} \\ 
    \midrule
    UpDn\cite{anderson2018bottom}& - & 39.96 & 43.01 & 12.07 & 45.82 & 63.48 & 81.18 & 42.14 & 55.66\\
    \midrule
    AdvReg\cite{SainandanRamakrishnan2018OvercomingLP} & UpDn & 41.17 & 65.49 & 15.48 & 35.48 & 62.75 & 79.84 & 42.35 & 55.16\\
    RUBi\cite{cadene2019rubi} & UpDn & 44.23 & 67.05 & 17.48 & 39.61 & - & - & - & - \\
    LMH\cite{clark2019don} & UpDn & 52.01 & 72.58 & 31.12 & 46.97 & 56.35 & 65.06 & 37.63 & 54.69\\
    % MCE\cite{Clark_Yatskar_Zettlemoyer_2020}
    % LBCL\cite{Lao_Guo_Liu_Chen_Pu_Lew_2021} &  & 52.01 & 72.58 & 31.12 & 46.97 & 56.35 & 65.06 & 37.63 & 54.69 & 55.27\\
    AdaVQA\cite{guo2021adavqa} & UpDn & 54.67 & 72.47 & 53.81 & 45.58 & - & - & - & - \\
    % IntroD\cite{NEURIPS2021_878d5691} & CSS & 60.17 & 89.17 & 46.91 & 48.62 & 62.57 & 78.57 & 41.42 & 56.00 \\
    CF-VQA\cite{niu2021counterfactual} & UpDn & 53.55 & {\textbf{91.15}} & 13.03 & 44.97 & {63.54}& 82.51 & {43.96} & 54.30 \\
    CIKD\cite{Pan_Li_Zhang_Tang_2021} & UpDn & 54.05 & 90.01 & 15.10 & 45.88 & 61.29 & 76.34 & 40.20 & 55.43 \\
    GGE\cite{han2021greedy} & UpDn & 57.32 & 87.04 & 27.75 & 49.59 & 59.11 & 73.27 & 39.99 & 54.39\\
    D-VQA\cite{wen2021debiased} & UpDn & 61.91 & 88.93 & 52.32 & 50.39 & {64.96} & 82.18 & 44.05 & 57.54 \\
    GGD\cite{han2023general} & UpDn & 59.37 & 88.23 & 38.11 & 49.82 & 62.15 & 79.25 & 42.43 & 54.66\\
    GenB\cite{Cho_Kim_Ryu_Kweon_2023}& UpDn & 59.15 & 88.03 & 40.05 & 49.25 & 62.74 & \textbf{86.18} & 43.85 & 47.03\\    
    % SAN\cite{ZichaoYang2016StackedAN} & 24.96 & 38.35 & 11.14 & 21.74 & 52.41 & 70.06 & 39.28 & 47.84 & 32.50\\
    % S-MRL\cite{cadene2019rubi} & 38.46 & 42.85 & 12.81 & 43.20 & 63.10 & -     & -     & -     & 36.38\\
    \midrule

    Template-based\cite{Kafle_Yousefhussien_Kanan_2017} & UpDn & 39.75 & 43.03 & 14.98 & 44.83 & 63.83 & 81.61 & 41.98 & 56.10 \\ 
    ASL\cite{Teney_Hengel_2019} & UpDn & 46.00 & 58.24 & 29.49 & 44.33 & - & - & - & - \\
    CVL\cite{Abbasnejad_Teney_Parvaneh_Shi_Hengel_2020} & UpDn & 42.12 & 45.72 & 12.45 & 48.34 & - & - & - & -  \\
    GradSup\cite{Teney_Abbasnedjad_van_den_Hengel_2020} & UpDn & 46.80 & 64.50 & 15.30 & 45.90 & - & - & - & -  \\
    % VGQE\cite{KV_Mittal_2020} & UpDn & 
    RangImg\cite{Teney_Kafle_Shrestha_Abbasnejad_Kanan_Hengel_2020} & UpDn & 55.37 & 83.89 & 41.60 & 44.20 & 57.24 & 76.53 & 33.87 & 48.57  \\
    % VILLA\cite{Gan_Chen_Fu_Zhu_Cheng_Liu_2020} & - & - & - & - & - & 74.69 & 
    CSS\cite{chen2020counterfactual} & UpDn & 58.95 & 84.37& 49.42& 48.24 & 59.91 & 73.25 & 39.77& 55.11 \\
    Mutant\cite{gokhale2020mutant} & UpDn & 61.72 & 88.90 & 49.68 & 50.78 & 62.56 & 82.07 & 42.52 & 53.28 \\
    % mango\cite{Li_Gan_Liu_2020} swapmix\cite{Gupta_Li_Kortylewski_Zhang_Li_Yuille_2022}
    Unshuffling\cite{Teney_Abbasnejad_van_den_Hengel_2021} & UpDn &  43.47 & 47.82 & 14.35 & 49.18 & 43.47 & 81.99 & 43.07 & 55.21 \\
    SimpleAug\cite{Kil_Zhang_Xuan_Chao_2021} & UpDn & 52.65 & 66.40 & 43.43 & 47.98 & 64.34 & 81.97 & 43.91 & 56.35 \\
    ECD\cite{Kolling_More_Gavenski_Pooch_Parraga_Barros_2022} & LMH & 59.92 & 83.23 & 52.29 & 49.71 & 57.38 & 69.06 & 35.74 & 54.25 \\
    KDDAug\cite{chen2022rethinking}& UpDn & 60.24 & 86.13 & \textbf{55.08} & 48.08 & 62.86 & 80.55 & 41.05 & 55.18 \\
    % AttReg\cite{} & LMH & 60.00 & 86.80 & 51.63 & 48.25 & 62.74 & 79.71 & 41.68 & 55.42 & 62.25 \\
    AttReg\cite{Liu_Guo_Yin_Song_Liu_Nie_Zhang_2022} & UpDn & 46.75 & 66.23 & 11.94 & 46.09 & 64.13 & 81.72 & 43.77 & 56.13  \\
    \midrule
    SSL\cite{zhu2020overcoming} & UpDn & 57.59 & 86.53 & 29.87 & 50.03 & - & - & - & - \\
    CSS+CL\cite{Liang_Jiang_Hu_Zhu_2020} & UpDn & 40.49 & 42.90 & 12.44 & 46.93 & - & - & - & - \\
    MMBS\cite{Si_Liu_Meng_Lin_Fu_Cao_Wang_Zhou_2022} & UpDn & 48.19 & 65.00 & 14.05 & 48.75 & 63.84 & 79.61 & 44.23 & 57.05\\
    % VQA-BC\cite{Lao_Guo_Chen_Pu_Lew_2022} 
    LSP\cite{Liu_Fan_Zhou_Xu_2023} & UpDn & 61.95 & 89.50 & 52.44 & 50.12 & 65.26 & 82.38 & 44.77 & 57.67 \\
    DM\cite{wu2022overcoming}& UpDn & 61.13 & 88.13 & 45.98 & 51.13 & 63.53 & 81.09 & 39.61 & 56.52 \\
    
    \midrule
    GVQA\cite{agrawal2018don} & UpDn & 31.30 & 57.99 & 13.68 & 22.14 & 48.24 & 72.03 & 31.17 & 34.65 \\
    AttAlign\cite{selvaraju2019taking} & UpDn & 39.37 & 43.02 & 11.89 & 45.00 & 63.24 & 80.99 & 42.55 & 55.22 \\
    HINT\cite{selvaraju2019taking} & UpDn & 46.73 & 67.27 & 10.61 & 45.88 & 63.38 & 81.18 & 42.99 & 55.56 \\
    SCR\cite{wu2019self} & UpDn & 49.45 & 72.36 & 10.93 & 48.02 & 62.2 & 78.8 & 41.6 & 54.5\\
    DLR\cite{jing2020overcoming} & UpDn & 48.87 & 70.99 & 18.72 & 45.57 & 57.96 & 76.82 & 39.33 & 48.54\\
    % rankvqa\cite{Qiao_Yu_Liu_2020},simplereg\cite{Shrestha_Kafle_Kanan_2020}, RMFE\cite{Gat_Schwartz_Schwing_Hazan_2020} ,LP-Focal\cite{Lao_Guo_Liu_Lew_2021}, SAR\cite{si2021check}
    LPF\cite{Liang_Hu_Zhu_2021} & UpDn & 51.57 & 87.33 & 12.25 & 43.61 & 62.63 & 79.51 & 42.90 & 55.02 \\
    CCB\cite{Yang_Feng_Li_Shen_Wang_Jiang_2021} & UpDn & 57.99 & 86.41 & 45.63 & 48.76 & 60.73 & 78.37 & 36.88 & 53.17 \\ 
    Loss-Rescaling\cite{Guo_Nie_Cheng_Tian_Zhang_2022} & UpDn & 47.09 & 68.42 & 21.71 & 42.88 & 55.50 & 64.22 & 39.61 & 53.09 \\
    RMLVQA\cite{basu2023rmlvqa} & UpDn & 60.41 & 89.98 & 45.96 & 48.74 & 59.99 & 76.68 & 37.54 & 53.2 \\
    LRLGC\cite{cao2023overcoming}& UpDn & 60.91 & 89.95 & 45.13 & 50.03 & 60.81 & 77.65 & 39.25 & 53.71 \\

    \textbf{EBGDA+FAA(Ours)} & UpDn & {\textbf{62.78}} & 87.90 & {51.74} & \textbf{52.79} & \textbf{65.47} & 82.51 & \textbf{57.84} & \textbf{58.64}\\
    \bottomrule
    \end{tabular}
    }
\end{table*}


\subsection{Multimodal Debiasing Experiments}
Recent studies emphasize that VQA-CP v2 primarily addresses limited biases, particularly shortcuts between question types and answers. In contrast, VQA-VS reorganizes VQA v2 by incorporating three different forms of bias to tackle various incorrect correlations. Therefore, we conducted debiasing experiments on the VQA-VS benchmark, with the results presented in Table \ref{table vs}. 
% 表\ref{table vs}说明了DAP模型在VQA-VS下的表现，它在语言、视觉和多模态偏差方面表现出色，优于大多数基线模型。与预训练的LXMERT相比，DAP表现出相当的稳健性，有效地解决了视觉问答任务中的各种偏差。
Table \ref{table vs} illustrates the performance of the EBGDA+FAA model on the VQA-VS benchmark, where it demonstrates superior performance in addressing linguistic, visual, and multimodal biases, outperforming most baseline models. Compared to the pre-trained LXMERT model, EBGDA+FAA exhibits considerable robustness, effectively mitigating various biases in the VQA task.

\begin{table*}[ht]  % 表2 
\centering
\caption{Regarding the experimental outcomes of EBGDA+FAA on the VQA-VS dataset, we have presented the relevant experimental performance reports associated with this dataset. Each column displays the performance results of the corresponding best and second-performing models.}
\label{table vs}
\resizebox{\linewidth}{!}{%
\begin{tabular}{@{}cccccccccccc@{}}
\toprule
                        &                       & \multicolumn{10}{c}{VQA-VS OOD Test Sets}\\ \midrule
\multirow{2}{*}{Model} & \multirow{2}{*}{Base} & \multicolumn{4}{c}{Language-based}                                & \multicolumn{2}{c}{Visual-based} & \multicolumn{3}{c}{multi-modality}               & \multirow{2}{*}{mean} \\ \cmidrule(lr){3-11}
                        &                       & QT             & KW             & KWP            & QT+KW          & KO              & KOP            & QT+KO          & KW+KO          & QT+KW+KO       &                       \\ \midrule
UpDn &  & 32.43 & {45.10} & 56.06 & {55.29} & 33.39 & 41.31 & {46.45}  & 54.29 & {56.92} & 46.80                 \\
+LMH & UpDn  & {33.36}     & 43.97          & 54.76          & 53.23          & 33.72           & 41.39          & 46.15          & 51.14          & 54.97          & 45.85\\
BAN  & UpDn & \underline{33.75} & 46.64  & 58.36  & 57.11 & 34.56  & 42.45 & \underline{47.92} & 56.26  & \underline{59.77} & 48.53 \\ \midrule
LXMERT   & - & \textbf{36.46} & \textbf{51.95} & \textbf{64.17} & \textbf{64.22} & \textbf{37.69}  & \textbf{46.40} & \textbf{53.54} & \textbf{62.46} & \textbf{67.44} & \textbf{53.70}        \\ \midrule
\textbf{EBGDA+FAA(Ours)} & UpDn & {32.45} & \underline{47.83} & \underline{59.27} & \underline{60.59} & \underline{34.75} & \underline{43.98} & 44.47 & \underline{56.69} & 55.6 & \underline{51.33}\\ \bottomrule
\end{tabular}%
}
\end{table*}


% 4.4 节
\subsection{Analysis of Other Metrics}
In our approach, we aim to increase the role of visual content in reasoning. To assess its effectiveness, we use additional metrics.  In Table \ref{cgd}, we compare our results with other methods using the CGD metric. For a more detailed understanding of CGD, please refer to \cite{han2021greedy,shrestha2020negative}. Compared to GGE \cite{han2021greedy}, our approach performs better in terms of CGD, indicating improved utilization of visual information for answer prediction.

\begin{table}[ht]  % 表3
   \centering
   \caption{Experiment on the evaluation metric CGD using the FAA method on the VQA-CP v2 dataset. \textbf{Best} results are displayed in each column.}
   \label{cgd}
   \begin{tabular*}{\linewidth}{@{\extracolsep{\fill}}cccc}
        \toprule
         Method & CGR & CGW & CGD\\
        \midrule
        UpDn & 44.27 & 40.63 & 3.91 \\
        RUBi & 39.60 & 33.33 & 6.27 \\
        CSS & \textbf{46.70} & 37.89 & 8.87 \\
        \midrule
        GGE-DQ-iter & 44.35 & 27.91 & 16.44\\
        GGE-DQ-tog & 42.74 & \textbf{27.47} & 15.27\\
        \textbf{EBGDA+FAA(Ours)} & 45.09 & 27.54 & \textbf{17.56}\\ 
        \bottomrule
\end{tabular*}
\end{table}

% 4.5节
\subsection{Results On GQA-OOD and VQA-CE}
\begin{table}[t]
\centering
\caption{Experimental results on GQA-OOD and VQA-CE. Compared to the Best methods, EBGDA+FAA achieves the best performance on GQA-OOD. On VQA-CE, EBGDA+FAA also demonstrates the best performance in the Counter.}
\label{tab:gqa and ce}
\resizebox{\linewidth}{!}{%
\begin{tabular}{@{}cccccc@{}}
\toprule
\multirow{2}{*}{Model} & \multicolumn{3}{c}{GQA-OOD Test} & \multicolumn{2}{c}{VQA-CE} \\ 
\cmidrule(l){2-4} 
\cmidrule(l){5-6}
     & All    & Tail   & Head  & Counter    & Easy    \\ \midrule
UpDn & 46.87 & 42.13 & 49.16 & 33.91 & \textbf{76.69} \\ 
RUBi & 45.85     & 43.37     & 47.37    & 32.25         & 75.03      \\
LMH  & 43.96     & 40.73     & 45.93    & 34.26         & 73.13      \\ 
CSS   & 44.24     & 41.20     & 46.11    & 34.36         & 62.08      \\ 
RandImg & -         & -         & -        & 34.41         & 76.21      \\
Genb & 49.43 & 45.63 & 51.76 & 34.80 & 68.15 \\ \midrule
\textbf{EBGDA+FAA}  & \textbf{51.71}     & \textbf{46.54}     & \textbf{54.29}    & \textbf{35.93}         & 63.98      \\ \bottomrule
\end{tabular}%
}
\end{table}
GQA-OOD and VQA-CE are new datasets and evaluation methods for VQA debiasing. The table \ref{tab:gqa and ce} presents the experimental results of EBGDA+FAA. On GQA-OOD, EBGDA+FAA outperforms methods such as CSS, LMH, and RUBi. In VQA-CE, EBGDA+FAA achieves the highest accuracy in the Counter, validating its debiasing capability. These results strongly demonstrate the debiasing effectiveness of EBGDA+FAA.

\subsection{Abalation Experiments}
The ablation experiment section verifies the performance of the EBGDA and FAA components in the method.

\textbf{The effectiveness of EBGDA.} In this section, we conduct experiments using only the EBGDA component. The experimental results are shown in Table \ref{ebgda:baseline model}. By comparing EBGDA with methods that use the same baseline model, we can see that EBGDA also achieves performance improvements.
\begin{table}
\centering
\caption{Ablation experiments with EBGDA}
\label{ebgda:baseline model}
\resizebox{0.9\linewidth}{!}{
\begin{tabular*}{\linewidth}{@{\extracolsep{\fill}}lcccc}
\toprule
Method & All   & Yes/No & Num.  & Other \\ \midrule
UpDn & 39.96 & 43.01  & 12.07 & 45.82      \\
SAN & 38.65 & 40.09  & 12.98 & 44.67     \\ 
BAN & 35.94 & 40.39  & 12.24 & 40.51     \\ 
S-MRL & 38.46 & 42.85  & 12.81 & 43.20     \\ 
\textbf{EBGDA} & 60.10 & 86.99 & 36.72 & 52.42     \\ \bottomrule

\end{tabular*}}
\end{table}

\textbf{The effectiveness of FAA.} In this section, we conduct experiments using only the FAA component. The experimental results are shown in Table \ref{faa:baseline model}. When comparing FAA with methods using the same baseline model, we can observe that FAA also achieves performance improvements.

\begin{table} % 表6 
\centering
\caption{Ablation experiments with FAA}
\label{faa:baseline model}
\resizebox{0.9\linewidth}{!}{
\begin{tabular*}{\linewidth}{@{\extracolsep{\fill}}lcccc}
\toprule
Method & All   & Yes/No & Num.  & Other \\ \midrule
UpDn & 39.96 & 43.01  & 12.07 & 45.82      \\
SAN & 38.65 & 40.09  & 12.98 & 44.67     \\ 
BAN & 35.94 & 40.39  & 12.24 & 40.51     \\ 
S-MRL & 38.46 & 42.85  & 12.81 & 43.20     \\ 
\textbf{FAA} & 60.18 & 83.27 & 37.82 & 54.21     \\ \bottomrule
\end{tabular*}}
\end{table}


\textbf{Forgotten Sequence.} The concept of forgetting attention in this paper is based on the process of human answering relevant questions. The ablation experiment considers the order of forgetting in the algorithm to verify the validity of the concept.
Table \ref{table order} shows the effect of three different sequences of attention on the performance of the model:
\begin{table}[t] % 表7 
\caption{Impact of forgetten order on performance}
\label{table order}
\begin{tabular*}{\linewidth}{@{\extracolsep{\fill}}cccccc}
\toprule
    {} & {} & \multicolumn{4}{c}{VQA-CP v2 test}\\
    \cmidrule[0.5pt](rl){3-6}
    {Order} & {Base} & {All} & {Y/N} & {Num.} & {Other} \\ 
    \midrule
    FAA & UpDn & {60.74} & {83.99} & {41.45} & {53.85} \\
    Reverse & UpDn & 37.86 & {78.00} & 15.34 & 23.00 \\
    Linear & UpDn & {27.13} & {71.99} & {6.18} & 9.37 \\
    \bottomrule
\end{tabular*}
\end{table}
\begin{enumerate}[label=(\arabic*)]
\item In the method of this article, we follow the normal attention process, pay attention to the image areas that are most relevant to the problem, and forget the results obtained from the irrelevant areas.
\item In contrast to the normal attention flow, the model first notices the areas of the image that are less relevant to the problem, assigns higher weights to them, and finally forgets the areas of the image with lower weights.
\item The feature areas in the image are arranged in the original linear order, and the model forgets the relevant features in the same order.
\end{enumerate}
By comparing the model performance of different forgetting sequences, we were able to observe that the FAA achieved excellent performance by forgetting irrelevant regions, while the other forgetting sequences resulted in decreased performance. This suggests that the human attentional forgetting mechanism on which the FAA is based works.

\begin{table}[t]
\caption{Performance corresponding to different attention thresholds}
\label{table threshold}
\begin{tabular*}{\linewidth}{@{\extracolsep{\fill}}cccccc}
\toprule
    {} & {} & \multicolumn{4}{c}{VQA-CP v2 test}\\
    \cmidrule[0.5pt](rl){3-6}
    {Thresholds} & {Base} & {All} & {Y/N} & {Num.} & {Other} \\ 
    \midrule
    0.5 & UpDn & {56.84} & {83.29} & {43.87} & 46.54 \\
    0.6 & UpDn & {60.74} & {83.99} & {41.45} & {53.85} \\
    0.7 & UpDn & {59.75} & {82.85} & {35.21} & 54.38 \\
    dynamic & UpDn & {61.65} & {85.37} & {40.68} & 54.98 \\
    \bottomrule
\end{tabular*}
\end{table}

\textbf{Attention Threshold Selection.} In this method, we set thresholds to allow the model to filter image regions to determine which are forgotten and which are retained. Different threshold sizes make the model achieve different performance when forgetting the image region. If the threshold is too high, the model will forget most of the image region, resulting in the model being unable to obtain useful information from the image, while if the threshold is too low, the model will retain useless information, thus degrading the algorithm to a common attention algorithm. Therefore, this section sets up a comparative analysis of different threshold sizes to determine the appropriate parameter as the forgetting threshold.
As shown in Table \ref{table threshold}, the corresponding experimental results of the three threshold sizes selected by us are reported.

Considering the three different choices of attention threshold, in the original attention scheme of UpDn, the contribution degree of different image regions to the answer is realized by the assigned weight, whose value is between 0 and 1. Therefore, we choose the sizes of 0.5, 0.6 and 0.7 for experimental comparison. The final experimental results show that when the threshold size is 0.6, the experimental effect reaches the optimal performance, and the forgetting ability of the model reaches the equilibrium.
Additionally, we also present the approach using a dynamic threshold, where the threshold is treated as a hyperparameter and optimized during training.
\begin{table}[t] % 表 9
\caption{The impact of different feature combinations on VQA modeling.}
\label{table faster}
% \begin{tabular*}{\linewidth}{@{\extracolsep{\fill}}cccccc}
\resizebox{\linewidth}{!}{
\begin{tabular}{@{}cccccc@{}}
\toprule
Number  & Visual & Spatial & CLS & Attribute & All   \\ 
\midrule
1 &    \checkmark    &         &     &           & 56.34 \\
2 &    \checkmark    &    \checkmark    &     &           & 55.30 \\
3 &    \checkmark    &    \checkmark    &  \checkmark  &           & 54.53 \\
4 &    \checkmark    &    \checkmark    &  \checkmark  &     \checkmark     & 59.02 \\ 
\bottomrule
\end{tabular}
}
\end{table}

\textbf{Visual Information}. As shown in Table \ref{table faster}, In our method, the output after Faster RCNN detection is combined as the visual input of the model in this paper. Specifically, we fuse each of the four combinations as new visual feature inputs, confirming the benefit of diverse visual information in model comprehension.

\begin{table}[t]  % 表 10
\caption{The performance under different number of attention layers.}
\label{table layers}
% \begin{tabular*}{0.95\linewidth}{@{\extracolsep{\fill}}cccccc}
\resizebox{\linewidth}{!}{
\begin{tabular}{@{}cccccc@{}}
\toprule
Layers & All   & Yes/No & Num.  & Other & Time/Epoch \\ \midrule
1      & 56.95 & 83.77  & 42.99 & 46.72 & 1048s      \\
2      & {60.74} & {83.99} & {41.45} & {53.85 }& 1303s      \\
3      & 57.67 & 82.66  & 46.44 & 47.66 & 1380s      \\
4      & 58.56 & 80.81  & 50.68 & 47.26 & 1532s       \\
5      & 56.21 & 84.48  & 35.66 & 47.03 & 1540s      \\ \bottomrule
\end{tabular}
}
\end{table}

\textbf{Layers Of Attention}. As shown in Table \ref{table layers}, We set up different levels of attention in the method to perform validation experiments. Specifically, when humans answer questions by focusing on different areas in the image, they may go through multiple target shifts to determine the final area, while in the simulation of a computer, this operation can be achieved by setting the number of layers of attention. In this experiment, we set up a total of five different layers, as you can see the model performs the best with three attention layers. This configuration significantly improves performance during inference compared to fewer layers. However, increasing the number of layers yields a slightly worse overall accuracy as well as increasing inference time by nearly 200 seconds. Thus, we settle on three attention layers. Regarding accuracy degradation, we believe that this phenomenon arises due to the fact that the model recognizes incorrect visual information and masks relevant regions, thus hindering accurate answer retrieval.

\begin{table}[t]
\caption{Ablation experiments involving FAA and the CSS method}
\label{table qcss}
\begin{tabular*}{\linewidth}{@{\extracolsep{\fill}}ccccc}
\toprule
Method & All   & Yes/No & Num.  & Other \\ \midrule
Q-CSS & 56.19 & 80.83  & 40.33 & 47.63      \\
CSS   & 58.17 & 84.57  & 46.99 & 47.40     \\
FAA+Q-CSS & 58.31 & 80.83  & 48.90 & 49.10     \\ 
\textbf{FAA+CSS} & 60.09 & 88.55 & 53.16 & 47.09 \\ \bottomrule
\end{tabular*}
\end{table}

\textbf{Q-CSS}. In our approach, we opted for a single-word replacement strategy, combined with FAA. The experimental results in Table \ref{table qcss} encompass a partial replication of the Q-CSS strategy from the CSS method and the QV-CSS strategy, incorporating FAA into both Q-CSS and CSS. Notably, our approach exhibits approximately 2\% improvement in accuracy over Q-CSS and CSS.

\subsection{Qualitative results}
As depicted in Fig. \ref{visual}, the original image, after two rounds of attentional operations, masks out irrelevant areas based on attentional weights in the Fig. \ref{visual} (1), ultimately identifying the target region relevant to the answer. 

\begin{figure*}[t]  % 图 5.  4个子图
    \centering
    \includegraphics[height=0.37\textwidth, width=1.0\textwidth]{image/4个子图.pdf}
    \caption{The results of qualitative analysis show the flow of our model when making predictions by masking different image regions so that the model focuses on the effective ones}
    \label{visual}
\end{figure*}

In Fig. \ref{visual}, more examples are given to analyze the effect of forgotten attention on changes in image areas. For example, in the example of the Fig. \ref{visual} (2), the image of the animal is the area where the zebra is located, and there is overlap between some areas that are unrelated to the problem and the zebra, which is covered by the FAA to some extent, but most of the zebra area is still captured by the model. Similarly, in the Fig. \ref{visual} (3) and Fig. \ref{visual} (4), the areas of the sign is somewhat obscured, but the model still understands the semantics of the remaining areas of the image and gives the correct answer.

\section{Conclusion}
% In this paper, we introduce a novel attention mechanism, the Forget Attention Algorithm (FAA), aimed at mitigating language bias. We regard language bias as a lack of model comprehension of visual content. We artificially mask the image content in our method using a ``forgetting'' strategy, enabling the model to mimic human attention flow in each iteration for multi-step reasoning. We experiment with our method on datasets VQA v2, VQA-CP v2, and VQA-VS to validate its effectiveness.

% Our method has the following limitations: First, it only models and optimizes the inherent biases present in the training data, and it cannot ensure that all biases encountered in real-world scenarios are addressed. Second, the image masking approach may cause the model to make incorrect judgments early in the process, potentially leading to erroneous answers.
In this paper, we proposed a novel debiasing approach for VQA (VQA) by leveraging an ensemble model framework to address and mitigate various types of biases, including linguistic, visual, and multimodal biases. Through the integration of a bias detection mechanism, we dynamically identified and evaluated the impact of biases on the model's performance. By applying overfitting strategies selectively, the method enabled the main model to focus on unbiased and beneficial data, thus improving its generalization capability. Our experiments on VQA-CP v2, VQA v2, and VQA-VS benchmarks demonstrated state-of-the-art performance, especially in challenging scenarios involving complex multimodal correlations. This approach provides an effective solution for reducing bias in VQA tasks, leading to more robust and accurate predictions.

% While FAA effectively models and mitigates biases inherent in training data, two key limitations warrant discussion: 
% \begin{enumerate}[label=(\arabic*)]
% \item Scope of Bias Mitigation: The current framework optimizes for dataset-specific biases but lacks generalization guarantees for unseen real-world distribution shifts. 

% \item Error Propagation: The iterative masking process may amplify early-stage misjudgments due to incomplete visual context, potentially leading to cascading reasoning errors. 
% \end{enumerate}
% These limitations suggest promising directions for future research, including adversarial bias augmentation and uncertainty-aware masking strategies to enhance robustness.

\subsection*{Funding}
This work was supported by the National Natural Science Foundation of China (Grant No. 62172457), Science and Technology Research Project in Henan Province of China (Grant No. 242102210104 and 252102210032), Key Scientific Research Project in Henan University of China (Grant No. 25B520021).

\subsection*{Data Availability Statement}
The datasets supporting the findings of this work are publicly accessible to ensure reproducibility and further research. All the data used in this work are available in these collections: 
\href{https://phoebussi.github.io/VQA-VS-homepage/}{https://phoebussi.github.io/VQA-VS-homepage/},
\href{https://www.iro.umontreal.ca/~agrawal/vqa-cp/}{https://www.iro.umontreal.ca/agrawal/vqa-cp/},
\href{https://visualqa.org/index.html}{https://visualqa.org/index.html}.

\subsection*{Abbreviations}
The following abbreviations are used in this paper:\\
\noindent 
\begin{tabular}{@{}ll}
VQA & Visual Question Answering\\
FAA & Forgotten Attention Algorithm\\
EBGDA & Ensemble Bias Gradient Debiasing Approach\\
GRU & Gated Recurrent Unit\\
UpDn & Bottom-Up and Top-Down Attention Model
\end{tabular}


\bibliographystyle{unsrt}
\bibliography{mybib}

% % bibliography style is not predefined; authors can apply their own,
% % but it should be sent to matjaz.gams@ijs.si with the accepted paper
% \begin{thebibliography}{99}

%   % book
%   \bibitem{} Author (year) {\it Title of the book}, Publisher.

%   % journal paper
%   \bibitem{} Author (year) Title of the paper, {\it Title of the journal}, Publisher, pp. nn--mm.

%   % conference
%   \bibitem{} Author (year) Title of the paper, {\it Title of the proceedings}, Publisher, Location, pp. nn--mm.

% \end{thebibliography}

\end{document}
