\documentclass[english]{jnlp_3.8}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{array}
\allowdisplaybreaks

\usepackage{hangcaption_jnlp}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition

%%%%%\usepackage{subcaption}
\usepackage{booktabs}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{32}
\Number{3}
\Month{September}
\Year{2025}
%%%%\Issuetitle{「　　　　」特集号}

\received{2025}{1}{31}
\revised{2025}{4}{22}
\accepted{2025}{6}{4}



\setcounter{page}{859}



%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Vowel Articulation and Tongue Position\\in Vision Language Models}
\eauthor{Haruki Sakajo\affiref{Author_1} \and Yusuke Sakai\affiref{Author_1} \and Hidetaka Kamigaito\affiref{Author_1} \and Taro Watanabe\affiref{Author_1}} 

\eabstract{%
How human vocalizations are articulated can be described by analyzing the tongue position. Researchers have discovered this through lived experience and detailed observation, including by MRI. Using this knowledge and personal experience, teachers can understand and explain the relationship between tongue positions and vowels, thus helping language students to learn pronunciation. Our preliminary studies suggest that language models (LMs), trained on extensive data from the linguistic and medical fields, can explain the mechanisms of vowel pronunciation. However, it is unclear whether multimodal LMs, such as Large-scale Vision Language Models (LVLMs), sufficiently align textual information with visual information. From this, the research question arises: Do LVLMs associate real tongue positions with vowel articulation? To investigate whether visual information can help LVLMs understand vowel articulation based on tongue positions, this study created video and image datasets from real-time MRI samples. We discuss how LVLMs predict vowels by analyzing several experimental results. Our findings suggest that LVLMs can potentially identify the interrelationship between vowels and tongue positions when reference examples are provided, but have difficulty without them. LVLMs also perform better when inferring directly from visual information than when offered text descriptions before inferring.
}
\ekeywords{Language Models, Vowel Articulation, Tongue Position, Phonetics, Phonology}

\headauthor{Sakajo et al.}
\headtitle{Vowel Articulation and Tongue Position in Vision Language Models}

%%%%%%%%%%%%%%%%%%%
\affilabel{Author_1}{}{Nara Institute of Science and Technology}

%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle
\renewcommand{\thefootnote}{}% 番号書式を"空"にする
\footnote[0]{This paper is an extended version of our preliminary paper \cite{sakajo-etal-2025-tonguescape} accepted for presentation at the 2025 Annual Conference of the Nations of the Americas Chapter of the Association for Computational Linguistics (NAACL 2025). We revised the main paper to include more analyses, details, and discussions. In particular, we analyzed the failures in fine-tuning and conducted an additional experiment using Chain-of-Thought to discuss our research question in more detail.}
\renewcommand{\thefootnote}{\arabic{footnote}}% 元(算用数字)に戻す


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S1
\section{Introduction}
In phonetics, vowels are distinguished and described by focusing on tongue positions and lip shape.
Researchers have explained vowel formation based on specific tongue positions during articulation \cite{jonesEverymansEnglishPronouncing1917}.
Speakers become aware of these speech mechanisms through training, introspection, and observing visual information (e.g., MRI).
For example, when pronouncing the English word ``image,'' speakers can perceive and explain that the initial sound is produced by positioning the tongue forward and high in the mouth.
This method introduces the cardinal vowels, demonstrating a speaker's understanding of tongue positions through introspection and objective observations, such as MRI, and linking this understanding to vowel articulation.
This knowledge allows us to explain how to pronounce and describe linguistic phenomena to language learners.

Language Models (LMs) are trained on copious data, which allows some to acquire commonsense insights \cite{bian-etal-2024-chatgpt}, including physical commonsense \cite{Bisk2020}.
The training data for LMs includes linguistic and medical information, with our preliminary studies showing that LMs possess knowledge of vowel pronunciation and the correlation between vowels and tongue positions as textual knowledge (see Section \ref{sec:preliminary_studies}).
To determine if LMs comprehend articulation relative to the articulatory organs requires multimodal information.
Multimodal LMs capture textual information, images, videos, and audio \cite{zhou-etal-2023-rome,li2024llavanext-interleave,geminiteamGeminiUnlockingMultimodal2024}. Moreover, their application is expanding to more specialized fields, such as clinical tasks involving the detection and explanation of diseases from clinical images, such as CT and MRI scans \cite{yanWorseRandomEmbarrassingly2024,pal-sankarasubbu-2024-gemini}.
However, the alignment among modalities, such as images and text, is often weak in multimodal LMs, and it remains unclear whether these models truly understand such interactions \cite{caoUnderstandingAttentionVisionLanguage2022,kawaharazukaRealWorldRobotApplications2024,hayashi-etal-2024-towards}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F1
\begin{figure}[b]
\begin{center}
\includegraphics{32-3ia4f01.eps}
\end{center}
  \caption{An illustration of our research question}
  \label{fig:introduction}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


\emph{Do LVLMs align tongue positions with vowel articulation?}
The research question is illustrated in Figure \ref{fig:introduction}.
If LVLMs can align the visual information of articulatory processes with textual descriptions of phonetics, they could contribute to further analysis of linguistic phenomena, such as vowel harmony, and assist with teaching pronunciation.
For example, a language learner would benefit from a tongue-movement analysis that offers visual or textual feedback and guidance provided in their first language, as illustrated in Figure \ref{fig:application}.
An LVLM trained on multimodal data could also help to base articulation lessons on human physiology, which would promote greater understanding.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%F2
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia4f02.eps}
\end{center}
  \hangcaption{An example of potential applications for this research. If LVLMs can associate tongue positions with vowel articulation, they may provide pronunciation feedback to language learners by analyzing articulatory data obtained from MRI scans or ultrasound recordings.}
  \label{fig:application}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


In this study, we curated and annotated videos and images of tongue positions, as illustrated in Figure \ref{fig:dataset_example}, from an existing MRI dataset of real-time articulatory movements \cite{maekawaRealtimeMRIArticulatory2022}.
We examined whether LVLMs can reliably interpret vowel articulation based on tongue positions in a visual/video question-answering format.
We found that some models appear capable of predicting lingual positioning in images of each vowel utterance in a few-shot setting, whereas other models struggle to recognize them in zero-shot, few-shot, or fine-tuned settings.
We also found that Chain-of-Thought can reduce performance, suggesting that exclusively textual information functions as noise in this task. 


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F3
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia4f03.eps}
\end{center}
    \hangcaption{Examples of our dataset and what human speakers do when predicting vowels from real-time MRI scans, and how we constructed our dataset to replicate it. VowelVideo simulates the process by which humans predict vowels from the entire pronunciation. VowelImage simulates observing the moment of articulation. VowelImageWithGuide simulates examining tongue positions within the oral cavity.}
    \label{fig:dataset_example}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2
\section{Background and Related Works}
\label{sec:backgroung_related_works}

%%%%%
\paragraph{Tongue and vowel in linguistics.} 
In phonetics, vowels are described by the height and backness of the tongue body in the oral cavity.
Tongue positions are measured relative to vowel category norms rather than absolute spatial coordinates \cite{jonesEverymansEnglishPronouncing1917,knightCambridgeHandbookPhonetics2021a}. 
These characteristics describe several linguistic phenomena \cite{knightCambridgeHandbookPhonetics2021a} and are also adopted in the International Phonetic Alphabet (IPA) \cite{associationHandbookInternationalPhonetic1999}.
For example, when pronouncing the vowel /i/ in Figure \ref{fig:introduction}, the tongue is moved forward and raised higher, creating a narrow space in the anterior region of the oral cavity.

The five-vowel system (/a/, /i/, /u/, /e/, /o/), which is used in Japanese, is one of the most common vowel inventories in world languages  \cite{kubozonoHandbookJapanesePhonetics2015}, and more than half of languages have vowel inventories this size \cite{wals-vowel-inventories}.
These vowels are characterized by tongue height and backness: high-mid-low and front-back\footnote{Vowel height can be explained as closeness. In this study, we use high-mid-low because preliminary studies show that LMs explain vowels using height.}.
The tongue positions of each vowel are \mbox{/a/ (low-back), }/i/ (high-front), /u/ (high-back), /e/ (mid-front), and /o/ (mid-back), as shown in Figure \ref{fig:japanese_vowel}.
The tongue position of the vowel /o/ can be determined as an intermediate position between /a/ and /u/.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%F4
\begin{figure}[b]
\begin{center}
\includegraphics{32-3ia4f04.eps}
\end{center}
    \caption{A chart of Japanese vowels}
    \label{fig:japanese_vowel}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


Speakers are typically unaware of their tongue movement during speech.
However, they can learn to modulate vowel articulation, for example, by producing sounds between /a/ and /e/.
This enables us to explain linguistic phenomena related to tongue positions, such as vowel harmony and sound change \cite{Bybee_2015}.
However, studying these phenomena sometimes requires introducing new but consistent principles \cite{ko-2012-tongue,joseph-2018-historical,wang-2020}.
If LVLMs can understand the articulation, they can provide valuable research insights into these underlying principles.

%%%%%
\paragraph{Tongue and vowels in language models.}
Several studies have focused on vowels and tongue-related information.
\citeB{steuer-etal-2023-information} found that phoneme-level LMs capture vowel harmony, a linguistic phenomenon related to tongue position.
\citeB{cotterell-eisner-2017-probabilistic} predicted vowel inventories by introducing models based on stochastic point processes.
They used formant information to identify vowels influenced by tongue position.
\citeB{cotterell-eisner-2018-deep} also used formant information to describe phonetic-related linguistic typology provided by a generative probability model.
However, since these studies focused on textual information, it remains unclear whether multimodal LMs can achieve an embodied understanding of the relationship between tongue positions and vowels using aligned textual and visual information.


%%%%%
\paragraph{Applications of tongue position analysis.}
Tongue positions are also used to improve speech synthesis systems.
\citeB{abeysingheVisualisingModelTraining2022} found that the vowel space of a speech synthesis model changes during fine-tuning and can be visualized.
\citeB{wuDeepSpeechSynthesis2023} introduced a speech synthesis method that uses MRI-based features and demonstrates that MRI provides helpful features for synthesizing speech.
If LVLMs can comprehend the relationship between tongue position and articulation, it would further advance research, contributing to the synthesis of fluent or intentionally disfluent speech, as observed in human speech.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3
\section{Preliminary Studies}
\label{sec:preliminary_studies}
We investigated whether LMs know the relationship between vowels and tongue positions using GPT-4o \cite{gpt4o} by asking simple questions.
Table \ref{tab:explanation_vowel_tonge} demonstrates that they should be able to coherently explain the relationship between tongue position and vowel articulation like human speakers.
Furthermore, Table \ref{tab:prediction_vowel_tonge} shows that they can also predict a vowel from the given tongue position.
We used each of the five vowels? tongue positions as a query, and GPT-4o answered by identifying the correct vowel.
The preliminary studies show, therefore, that LMs know how to pronounce vowels.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T1
\begin{table}[t]
\input{04table01.tex}
    \hangcaption{A sample explanation of a vowel /i/ by GPT-4o. LMs can explain how to pronounce a vowel, focusing on tongue positions and lip shape. The explanation is linguistically correct, and speakers can pronounce the vowel by following this instruction.}
    \label{tab:explanation_vowel_tonge}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T2
\begin{table}[t]
\input{04table02.tex}
    \hangcaption{A sample vowel prediction from the tongue position given by GPT-4o. LMs appear to have the capability of predicting vowels from textual information by using their textual knowledge.}
    \label{tab:prediction_vowel_tonge}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4
\section{Dataset: Tonguescape}
\label{sec:dataset}
We created the Tonguescape dataset, based on the Real-time MRI Articulatory Movement Database (rtMRIDB) \cite{maekawaRealtimeMRIArticulatory2022} that contains video recordings of Japanese phoneme articulations.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.1
\subsection{Real-time MRI Articulatory Movement Datasets}
The rtMRIDB comprises utterances by 22 Japanese speakers with physiological variations.
It captures the lateral view of the speech production process, including tongue and pharyngeal movements.
Each video comprises sequential MRIs and corresponding audio files.
The data were recorded at either 14 or 27 frames per second (FPS).
Each video begins with the resting position, which is maintained during non-speaking states, and concludes with the resting position.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.2
\subsection{Tonguescape}
When predicting vowels, subjects who analyze the real-time MRI will (1) detect the moment of vowel articulation in the video, (2) observe the tongue and determine its position, and then (3) select a vowel.
We proposed a QA dataset for vowel prediction from real-time MRI recordings of tongue movements during vowel articulation. It comprised three steps, each corresponding to the stages of human perception as shown in Figure \ref{fig:dataset_example}.
We curated and annotated videos and images of tongue positions from the rtMRIDB, which comprises real-time MRI recordings of articulatory movements of Japanese phonemes.
This study focused on the Japanese five-vowel system (/a/, /i/, /u/, /e/, /o/), in which every vowel has distinct articulatory features.
Through this process, we created three datasets, VowelVideo, VowelImage, and VowelImageWithGuide.
We used five samples as validation data for fair comparison across three datasets and to change the prompts.

%%%%%
\paragraph{VowelVideo}
We extracted 120 silent real-time MRI videos from the rtMRIDB in which the five basic isolated vowels (/a/, /i/, /u/, /e/, /o/) were uttered. We split the data into five samples (one for each vowel) as training data, five samples as validation data, and the remaining 110 samples as test data. Since the tongue position during vowel articulation is not significantly affected by the pronunciation of preceding consonants, we also curated 1,653 videos where a vowel follows a consonant, such as in /ka/ and /na/, as additional training data, totaling 1,658 videos.
These additional training data are videos that record both a consonant and the following vowel.
The test and validation data comprise videos of isolated vowel pronunciations.
The videos capture the entire articulation process, starting from a non-speaking state, progressing through the articulation, and returning to a non-speaking state.
Each video is approximately 1 or 2 seconds long and consists of 14 or 27 frames per second.

%%%%%
\paragraph{VowelImage}
As shown in Figure \ref{fig:dataset_example}, we manually selected one of the most characteristic and representative frames from each of the five basic isolated vowels in the VowelVideo dataset. This allowed us to separate the process of selecting representative moments from the video to estimate vowels based on the characteristics of vowel images. 

%%%%%
\paragraph{VowelImageWithGuide}
Several studies show that adding markers as guides in the images improved the performance for various vision-understanding tasks \cite{shtedritski2023does}.
Inspired by this work, we added guide markers to all images in the VowelImage dataset to facilitate identifying the tongue position within the oral cavity, similar to human perception, as shown in Figure \ref{fig:dataset_example}. In the images, we drew ellipses as simple guides within the oral cavity to encompass the palate, the body of the tongue during low vowel articulation, the tip of the tongue during front vowel articulation, and the root of the tongue during back vowel articulation. These guides were automatically drawn at similar coordinates across all images in the dataset and were manually checked.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%T3
\begin{table}[t]
\input{04table03.tex}
    \caption{Dataset statistics}
    \label{tab:dataset_stats}
{\vskip -7pt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.3
\subsection{Dataset Statistics}
Table \ref {tab:dataset_stats} shows the dataset statistics.
The VowelImage and VowelImageWithGuide datasets each comprised 120 videos or images.
From these, we selected five samples (one for each vowel) as training data, another five samples as development data, and used the remaining 110 samples as test data.
The training and development data were from a single participant each, and the data from these participants were excluded from the test data.
VowelVideo comprises 1,773 videos.
The amounts of development and test data equaled those used in VowelImage and VowelImageWithGuide.
The remaining vowel videos were used as the test datasets.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.4
\subsection{Dataset Difficulty: Hypotheses}
\label{sec:dataset_dataset_difficulty}
Comparing the difficulty levels of VowelVideo and VowelImage is challenging.
We propose two hypotheses regarding the difficulty of the datasets.

%%%%%
\paragraph{VowelImageVideo and VowelImage}
VowelVideo is more challenging than VowelImage because detecting a frame where vowels are pronounced from the sequence of frames appears to be difficult for LVLMs.

%%%%%
\paragraph{VowelImage and VowelImageWithGuide}
VowelImage is more difficult than VowelImageWithGuide because the markers are helpful during inference \cite{shtedritski2023does}.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5
\section{Experiment}
\label{sec:experiment}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.1
\subsection{Experimental Settings}
We investigated the ability of LVLMs to predict vowels from input video/images using our dataset. 
This reveals whether LVLMs associate tongue positions with vowel articulations because they had to consider tongue positions in the given video/images to predict correct vowels.
\mbox{Figure \ref{fig:experiment_example}} shows the prompt and an example from our experiments\footnote{Respectively, we used one, two, or four NVIDIA RTX A6000, NVIDIA A100, or NVIDIA GeForce RTX 3090 GPUs.}.
Our prompt was designed using \citeB{hu-levy-2023-prompting}.
As baselines, we employed CLIP \cite{radford2021learningtransferablevisualmodels} and fine-tuned CLIP (FT) with the VowelImage training set. CLIP is an image encoder model we used to encode the MRI images and classify them into vowels.
A linguist created the human baselines for all datasets. We use accuracy as a metric for evaluation.
We chose models that perform well in Visual Question Answering tasks, such as MMMU \cite{yue2023mmmu}, for each dataset. We also chose models used for clinically detecting and diagnosing diseases from CT and MRI medical images  \cite{yanWorseRandomEmbarrassingly2024,pal-sankarasubbu-2024-gemini}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%F5
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia4f05.eps}
\end{center}
    \hangcaption{Sample Instruction: We instructed the LVLMs to predict the vowels from an input video/image. We evaluated whether the predicted vowels matched the vowels corresponding to the data used as the input. We specified the vowels (/a/, /i/, /u/, /e/, and /o/).}
    \label{fig:experiment_example}
{\vskip -6pt}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.1.1
\subsubsection{VowelVideo}
We used six models: GPT-4o \cite{gpt4o}, Gemini 1.5 Pro \cite{geminiteamGeminiUnlockingMultimodal2024}, LLaVA-NeXT-Interleave \cite{li2024llavanext-interleave}, Phi-3.5-vision-instruct \cite{phi3}, VideoLLaMA2 \cite{damonlpsg2024videollama2}, and Qwen2-VL-Instruct \cite{Qwen2VL}. We also fine-tuned VideoLLaMA2 (FT) using VowelVideo training data with LoRA \cite{hu2022lora}.
The details of these models are given in Table \ref{tab:model_detail}.
The fine-tuning hyperparameters are given in Table \ref{tab:experiment_param_vieollama2_qwenvl}.

The 14- and 27-FPS videos were respectively slowed to 1 and 2 FPS for input to Gemini 1.5 Pro \cite{geminiteamGeminiUnlockingMultimodal2024}.
Due to its limitations, VideoLLaMA2 uses only 16 frames per video, whereas all frames were used for GPT-4o, LLaVA-NeXT-Interleave, and Phi-3.5-vision-instruct.
When using Qwen2-VL-Instruct, the input video was processed at 14 FPS regardless of its original recording rate.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%T4
\begin{table}[t]
\input{04table04.tex}
    \hangcaption{The model details. The Model ID is a Huggingface Repository ID or a code defined in the OpenAI or Gemini APIs.}
    \label{tab:model_detail}
{\vskip -3pt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%T5
\begin{table}[t]
\input{04table05.tex}
    \caption{Hyperparameters to fine-tune CLIP, VideoLLaMA2, and Qwen-VL-Chat}
    \label{tab:experiment_param_vieollama2_qwenvl}
{\vskip -5pt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


When using fine-tuned VideoLLaMA2, we only used an output predicted vowel for evaluation if it was part of a consonant-vowel pair.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.1.2
\subsubsection{VowelImage and VowelImageWithGuide}
We used GPT-4o, Gemini 1.5 Pro, LLaVA-NeXT-Interleave, Phi-3.5-vision-instruct, VideoLLaMA2, LLaVA-Med \cite{li2023llavamed}, Qwen-VL-Chat \cite{Qwen-VL}, and Qwen2-VL-Instruct. We also fine-tuned Qwen-VL-Chat (FT) using the VowelImage training set.
The fine-tuning hyperparameters are given in Table \ref{tab:experiment_param_vieollama2_qwenvl}.
The models? details are given in Table \ref{tab:model_detail}.
We conducted evaluations under 0-shot, 1-shot, and 5-shot settings. 
Note that we did not apply the 1-shot and 5-shot settings to VieoLLaMA2 and LLaVA-Med because they are unsuitable for providing a question-answer example with an image.

%%%%%
\paragraph{Zero-shot Setting}
Each image in the dataset was input to the LVLMs.
When using GPT-4o and Gemini 1.5 Pro, we specified a JSON schema, such as \{``vowel'': str\}, to output only the predicted vowel.
If the output was not only single vowels, we extracted the first vowel surrounded by `` '' or / / and considered it the predicted vowel.
We treated the model as having refused to answer the question if no vowel was surrounded by`` '' or / /.

%%%%%
\paragraph{One-shot Setting}
One training data sample was selected as a one-shot example.
For each training data sample, we used all the test data and evaluated the accuracy.
We gave the models the one-shot example using a conversation template and followed the instructions.
We used the conversation template defined in each model except Gemini 1.5 Pro.
For Gemini 1.5 Pro, we added the sample to the prompt.
We did not apply this setting to VideoLLaMA2 and LLaVA-Med because they were unsuitable for providing an image with a question-answer example.

%%%%%
\paragraph{Five-shot Setting}
All of the training data were used as five-shot examples, such as in Figure \ref{fig:five_shot_example}, meaning that an image corresponding to each of the five vowels was used.
We provided the models with five-shot examples using a conversation template, following the instructions, and presented in the order of /a/, /i/, /u/, /e/, and /o/.
We also use the conversation template defined in each model except Gemini 1.5 Pro.
In Gemini 1.5 Pro, we add examples in the prompt.
The order of the five-shot examples is the same in all models: /a/ image, /i/ image, /u/ image, /e/ image, and /o/ image.
We did not apply this setting to VideoLLaMA2 and LLaVA-Med because they are unsuitable for providing question-answer examples with images.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%F6
\begin{figure}[!t]
\begin{center}
\includegraphics{32-3ia4f06.eps}
\end{center}
    \caption{Five-shot examples of VowelImage.}
    \label{fig:five_shot_example}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.2
\subsection{Few-shot Examples as Relative Positions of the Tongue: Bridging Linguistics}
\label{sec:few-shot-example-as-relative-position}
We divided our research question into two aspects: 1) concerning \emph{absolute position} and 2) concerning \emph{relative position}.
The differences between these two positions are shown in Figure~\ref{fig:overview_probing}.
This is based on a finding in linguistics that vowels can be distinguished by considering the relative position of the tongue \cite{jonesEverymansEnglishPronouncing1917,knightCambridgeHandbookPhonetics2021a}.
``Absolute position'' in Figure \ref{fig:overview_probing}(a) means the actual tongue position in the oral cavity when pronouncing a vowel.
``Relative position'' of Figure \ref{fig:overview_probing}(b, c) is the tongue position in the oral cavity when pronouncing a vowel and then pronouncing another vowel.
In this context, there are two types of ``relative position'': the relative positions when compared with uttering another vowel and the relative positions arising from physiological differences between speakers.
Dividing it in this linguistics-based way allows a more detailed understanding of LVLMs? capabilities.
Understanding absolute positions means knowing the relationship between tongue positions and vowels and being able to distinguish the tongue position in an image and associate it with a specific vowel.
Understanding relative positions means understanding the relationship between each vowel and applying the knowledge of phonetics to predict vowels.
We evaluated the capability of recognizing relative positions using a few-shot example.


%%%%%%%%%%%%%%%%%%%%%%%%%%%F7
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia4f07.eps}
\end{center}
    \hangcaption{An overview of our methods. The ``Absolute Position'' (a) is an actual tongue position in the oral cavity. The ``Relative Position'' (b, c) is the tongue position at the moment of pronouncing a vowel with regard to the tongue positions for other vowels.}
    \label{fig:overview_probing}
\end{figure} 
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%
\paragraph{Relative Position in a One-shot Example}
The LVLMs' understanding of the relationship between relative tongue positions and vowels was evaluated by examining their one-shot performance 
Speakers can specify vowels from an MRI by referencing their own pronunciation experience and knowledge of tongue positions during speech, including information learned from MRIs and illustrations.
However, LVLMs appear to have little knowledge about tongue positions in the oral cavity.
Providing reference information about the tongue position of a vowel through a one-shot example is one of the approaches to bridge this gap, as shown in Figure \ref{fig:overview_probing}(b).
If an image indicates a vowel, the LVLMs can determine the tongue position using the given image.

%%%%%
\paragraph{Relative Position in Five-shot Examples}
The capability of understanding the relationship between tongue positions and vowels was evaluated by examining the fine-tuned models' performance and the LVLMs' few-shot performance.
If there were more than five images containing each of the five vowels, they determined which vowel the given image approximated, as in Figure \ref{fig:overview_probing}(c).
This approach enabled us to simulate the process by which human speakers predict vowels.

%%%%%
\paragraph{Relative Positions in Video}
We observed the relative positions in VowelVideo by looking at the tongue movements mentioned in Section \ref{sec:dataset}.
The LVLMs processed the video as a sequence of frames.
In contrast to a few-shot example, this allowed us to determine relative tongue positions at the moment of articulation within an utterance.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.3
\subsection{Results}

%%%%%
\paragraph{VowelVideo}
Table \ref{tab:result_accuracy_vowel} shows that the models struggle to predict vowels from videos correctly.
VideoLLaMA2 (FT) outperforms the original model, but its accuracy remains close to the level of chance.
This suggests they are not well aligned to identify vowels using video or sequential frame data.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%T6
\begin{table}[b]
\input{04table06.tex}
    \hangcaption{Vowel prediction accuracy (\%) of different models in three settings. The accuracy was calculated by using the number of correct predictions and the number of test data samples.}
    \label{tab:result_accuracy_vowel}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%
\paragraph{VowelImage}
As shown in Table \ref{tab:result_accuracy_vowel}, the accuracy of each model is approximately 20\% in the zero-shot setting.
The accuracy improves in the one-shot and five-shot settings, particularly with Qwen2-VL-72B-Instruct.
This suggests that while some LVLMs can infer vowels by considering relative tongue positions, they lack a linguist's understanding of absolute positions.

%%%%%
\paragraph{VowelImageWithGuide}
Table \ref{tab:result_accuracy_vowel} shows that GPT-4o and Qwen2-VL-72B-Instruct performed much better on five-shot settings than on the VowelImage datasets.
However, other LVLMs still faced challenges in predicting vowels from images with guides.
This suggests that the LVLMs struggled to infer vowels grounded in vision information.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6
\section{Discussions and Analysis}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.1
\subsection{Dataset Difficulty: Discussion}
\label{sec:comparison_dataset}

From Table \ref{tab:result_accuracy_vowel}, we can address the two hypotheses proposed in Section \ref{sec:dataset_dataset_difficulty}.

%%%%%
\paragraph{VowelVideo and VowelImage}
Table \ref{tab:result_accuracy_vowel} shows that, contrary to our hypothesis, it is easier for some models and humans to predict vowels from videos than from images under the zero-shot setting.
The video data contains the moment of articulation and the states before and after articulation.
For example, we can look at relative positions during pronunciation.
This is similar to using VowelImage in one-shot or five-shot settings.
Considering that some models exhibited similar accuracies on the VowelVideo and VowelImage datasets under one-shot or five-shot settings, this information helps to predict vowels, although it could also be due to noise.
Because some models and humans predict vowels more accurately from videos than images, it suggests that they utilize this information more effectively.
Furthermore, these results demonstrate that these models can consider relative positions.

%%%%%
\paragraph{VowelImage and VowelImageWithGuide}
In Table \ref{tab:result_accuracy_vowel}, following the guideline causes poor performance in both zero-shot and one-shot settings when using GPT-4o, VideoLLaMA2, LLaVA-Med, and Qwen2-VL-7B-Instruct, while the performances improve or remain unchanged when using the other models.
These results partially support our hypothesis.
The five-shot setting also has the same effect when using LLaVA-NeXT-Interleave, Qwen-VL-Chat, Qwen-VL-Chat (FT), and Qwen2-VL-7B-Instruct.
This suggests that the guideline may produce noise for some models while being helpful for others.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.2
\subsection{Dataset Difficulty for each LM}
Table \ref{tab:result_accuracy_vowel} demonstrates the dataset difficulty for LVLMs.
We can broadly divide the LVLMs used in the experiments into two groups: one demonstrating relatively high accuracy on at least one dataset/setting, and the other performing at or below the level of chance.
The first group included GPT-4o, Gemini 1.5 Pro, VideoLLaMA2 (FT), Qwen2-VL-7B-Instruct, and Qwen2-VL-72B-Instruct.
The other models were in the second group.
This starkly contrasted the difficulty of certain datasets for each group.
This also suggests that the difference between these two groups derives from the backbone LMs' training processes or training datasets, not the parameter size.
This is because, for example, Qwen2-VL-7B-Instruct and Qwen-VL-7B-Chat had the same number of parameters and performed differently.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.3
\subsection{CLIP, LMs, and Tongue Positions}
\label{sec:clip_lm_tongue_positions}
VowelImage and VowelImageWithGuide show that under the zero-shot setting, most models performed worse than CLIP Table \ref{tab:result_accuracy_vowel}.
However, under the five-shot setting, some models outperformed the fine-tuned CLIP.
The LVLMs' poor performance compared with CLIP under the zero-shot setting suggests that LVLMs have difficulty predicting vowels based on absolute positions and understanding the association between vowels and tongue positioning in their training techniques.
In contrast, the five-shot enhancement results imply that they can consider relative positions.
Analyzing the performance compared to CLIP reveals the ability of LVLMs to understand absolute and relative positions.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%F8
\begin{figure}[b]
\begin{center}
\includegraphics{32-3ia4f08.eps}
\end{center}
    \hangcaption{Confusion matrices of results using VowelImage and GPT-4o under the one-shot setting. Each panel represents a confusion matrix obtained when the vowel images at the top of the figures are given as one-shot examples.}
    \label{fig:results_one_shot_gpt4o}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.4
\subsection{Case Study: Analysis of the Results}
\label{sec:case_study_analysis}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.4.1
\subsubsection{Results of GPT-4o}
\label{sec:detail_gpt4o}

%%%%%
\paragraph{Zero-shot Setting}
The zero-shot setting results suggest that it is challenging for the model to read the absolute tongue positions and predict vowels.

%%%%%%
\paragraph{One-shot Setting}
As shown in Figures \ref{fig:results_one_shot_gpt4o} and \ref{fig:results_one_shot_guide_gpt4o}, the one-shot results show the proportions of inferred vowels for each expected vowel when using GPT-4o.
The results demonstrate that the model infrequently classified the image as the vowel provided as a one-shot example, suggesting that the model strictly considered positional similarity.
This suggests two key points: understanding tongue position variability and interpreting accuracy under the one-shot setting. 
The model did not seem to comprehend that there was some degree of freedom in tongue positioning during articulation.
If it did, it would likely estimate the one-shot vowel to be that with a tongue position closest to the provided image.
This effectively reduced the task from a five-choice problem to a four-choice problem.
Thus, the accuracy improvement can be attributed to the reduced choices rather than the model's understanding of relative tongue positions.
This result suggests that it struggled to recognize relative tongue positions given only one image.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%F9
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia4f09.eps}
\end{center}
    \hangcaption{Confusion matrices of results using VowelImageWithGuide and GPT-4o under the one-shot setting. Each panel represents a confusion matrix obtained when the vowel images at the top of the figures are given as one-shot examples.}
    \label{fig:results_one_shot_guide_gpt4o}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%
\paragraph{Five-shot Setting}
Figure \ref{fig:result_gpt4o_five_shot_all} illustrates that the five-shot setting outputs are mainly /a / or /i /.
The model tended to output the high vowel /i/ when input images that indicate higher vowels (/i/, /u/) were provided, while it frequently output the low vowel /a/ for others.
The model seemed to recognize tongue height.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%F10
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia4f10.eps}
\end{center}
    \hangcaption{Confusion matrices of results of VowelImage and VowelImageWithGuide in GPT-4o under the five-shot setting.}
    \label{fig:result_gpt4o_five_shot_all}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.4.2
\subsubsection{Results of Qwen2-VL-72B-Instruct}

%%%%%
\paragraph{Zero-shot Setting}
Table \ref{tab:result_accuracy_vowel} indicates that this model performed worse than the baseline and random choice.
A reason is that it did not predict any vowels for some images.
Although this leads to poor performance, it indicates that the model accurately predicted vowels based on the given information.
However, even given these facts, the model struggled to predict vowels.

%%%%%
\paragraph{One-shot Setting}
In contrast to the results of GPT-4o shown in Figure \ref{fig:results_one_shot_gpt4o}, in most cases, this model predicted the vowel given as a one-shot example, see Figure \ref{fig:results_one_shot_qwen2vl72b}.
However, the model provided a vowel with the same backness property when given /e/ and /o/ as one-shot examples.
This reveals that the model considers the positions of the tongue.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%F11
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia4f11.eps}
\end{center}
    \hangcaption{Confusion matrices of results using VowelImage and Qwen2-VL-72B-Instruct under the one-shot setting.}
    \label{fig:results_one_shot_qwen2vl72b}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%
\paragraph{Five-shot Setting}
Figure \ref{fig:result_qwen2vl72b_five_shot_all} illustrates that most predictions are vowels /a/, /i/, and /u/.
In numerous instances, the model identified the vowels /e/ and /o/ as /a/. This misclassification could be attributed to both /e/ and /o/, like /a/, being categorized as non-high vowels.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%F12
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia4f12.eps}
\end{center}
    \hangcaption{Confusion matrices of results of VowelImage and VowelImageWithGuide for Qwen2-VL-72b-Instruct under the five-shot setting.}
    \label{fig:result_qwen2vl72b_five_shot_all}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


This indicates that this model struggles to compare multiple images.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.4.3
\subsubsection{Analysis of the Results of LLaVA-Med}
LLaVA-Med is trained on clinical data but struggles to predict vowels, as shown in Table \ref{tab:result_accuracy_vowel} and Figure \ref{fig:result_llava_med}.
A reason for this low accuracy is that it refused to answer.
For example, it refused to answer for 48 samples when using VowelImage.
This means it tried to predict vowels for 62 samples, and the percentage of correct answers to the number of attempted answers was 20.97 \%.
When using VowelImageWithGuide under the zero-shot setting, the predicted vowels were distributed differently from other models under the zero-shot setting.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%F13
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia4f13.eps}
\end{center}
    \hangcaption{Confusion matrices of results using VowelImage and VowelImageWithGuide in LLaVA-Med under the zero-shot setting. The rejected outputs were removed from this heatmap.}
    \label{fig:result_llava_med}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.5
\subsection{Potential of Unimodal Biases}
Figures \ref{fig:result_image_llavanext_zero_five_shot}, \ref{fig:result_image_phi35_five_shot}, and \ref{fig:result_image_qwenvl} show that some models output the same vowel for almost all images.
This could be because these models ignore the given images for the outputs.
This is known as ``unimodal biases'' \cite{chen-etal-2024-quantifying}, where multimodal language models tend to rely on only one modality for the output.
Such results imply that these models have unimodal biases on this task.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%F14
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia4f14.eps}
\end{center}
    \hangcaption{Confusion matrices of results using VowelImage and LLaVA-NeXT-Interleave under the zero-shot and five-shot settings.}\label{fig:result_image_llavanext_zero_five_shot}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%F15
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia4f15.eps}
\end{center}
    \hangcaption{Confusion matrices of results of VowelImage and VowelImageWithGuide in Phi-3.5-vision-instruct under the five-shot setting.}
    \label{fig:result_image_phi35_five_shot}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%F16
\begin{figure}[t]
\begin{center}
\includegraphics{32-3ia4f16.eps}
\end{center}
    \hangcaption{Confusion matrices of results using VowelImage and Qwen-VL-Chat under the zero-shot and five-shot settings.}
    \label{fig:result_image_qwenvl}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.6
\subsection{Analysis of Fine-tuning Failure}
\label{sec:analysis_failuer}
We fine-tuned VideoLLaMA2 and Qwen-VL-Chat, but they showed limited improvement or even decreased performance.
The limited improvement is because the training dataset is small, despite the decrease in loss for each model.
The decreased performance could be attributed to the fact that this model outputs one of two vowels after fine-tuning, whereas it previously output the same vowel before fine-tuning. The fine-tuning varies this model's output, which may have caused lower performance for VowelImageWithGuide.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.7
\subsection{Do LVLMs Consider Tongue Position?}
\label{sec:do_lms_consider_tongue_position}
Under the one-shot setting, GPT-4o tended to infer mid or low vowels for most images of mid vowels, while Qwen2-VL-72B-Instruct inferred high vowels as illustrated in Figures \ref{fig:results_one_shot_gpt4o} and \ref{fig:results_one_shot_qwen2vl72b}.
Figures \ref{fig:result_gpt4o_five_shot_all} and \ref{fig:result_qwen2vl72b_five_shot_all} illustrate that, under the five-shot setting, both GPT-4o and Qwen2-VL-72b-Instruct inferred /a/ for most images of mid vowels.
These findings demonstrate that the models consistently predicted vowels when analyzing images of mid vowels /e/ and /o/ in each setting.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.8
\subsection{Do LVLMs Detect Tongue Positions?}
\label{sec:lms_detect_tongue_positions}
Table \ref{tab:result_image_height_backness_gpt4o_five_shot} shows the results of predicting tongue height and backness both directly from images and by converting the estimated vowels to tongue positions, using GPT-4o and Qwen2-VL-72B-Instruct under the five-shot setting.
These results indicate that, although the model can predict vowels from specific images, it still struggles to predict tongue positions accurately.
These results indicate that LVLMs can consider relative tongue positions.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T7
\begin{table}[t]
\input{04table07.tex}
    \hangcaption{Tongue height and backness prediction accuracies (\%) of GPT-4o and Qwen2-VL-72B-Instruct under a five-shot setting. The column named ``H \& B'', which means ``Height and Backness'', shows that both ``Height'' and ``Backness'' are predicted correctly. GPT-4o (vowel) and Qwen2-VL-72B-Instruct (vowel) mean that height and backness were converted from the predicted vowels under the five-shot setting. For example, the H \& B are ``high'' and ``front'' if /i/ is predicted. The definitions of the tongue position for each vowel are in Section \ref{sec:backgroung_related_works}.}\label{tab:result_image_height_backness_gpt4o_five_shot}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.9
\subsection{Is Textual Information Important?}
\label{sec:cot}
As discussed in Section \ref{sec:clip_lm_tongue_positions}, models, such as GPT-4o and Qwen2-VL-72B-Instruct, can consider the relative tongue position compared with CLIP.
For further understanding, we conducted experiments using Chain-of-Thought (CoT) \cite{wei-etal-2022-chainofthought,kojima-etal-2022-large,zhang2024multimodal}.
This comprises two steps introduced in \citeB{zhang2024multimodal}: generating video/image descriptions with the given context and predicting vowels using both the given context and the generated descriptions.
As in Section \ref{sec:experiment}, five-shot examples are provided with a short description instead of repeating detailed explanations that match the output of this additional experiment.
Table \ref{tab:cot_results} shows the result when using CoT with GPT-4o and Qwen2-VL-72B-Instruct.
The accuracies with CoT are reduced compared with those without CoT, meaning that for LVLMs, excessive textual information can register as noise. 
The generated sentences shown in Table \ref{tab:example_cot} suggest that textual reasoning sometimes contains incorrect information, resulting in incorrect answers.
In this challenging Visual Question Answering task, LVLMs' reasoning should follow their internal processes, not human-readable texts.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T8
\begin{table}
\input{04table08.tex}
    \caption{The accuracy (\%) when using Chain-of-Thought (CoT)}
    \label{tab:cot_results}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%T9
\begin{table}[t]
\input{04table09.tex}
    \caption{An example of the image description in CoT for an image of the vowel /i/ by GPT-4o}
    \label{tab:example_cot}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S7
\section{Conclusion: Do LVLMs associate tongue positions with vowel articulation?}
To address our research question, we investigated the capability of LVLMs to understand the relationship between tongue positions and vowels.

As shown in Table \ref{tab:prediction_vowel_tonge}, LMs appear to understand the relationship between the positions of the tongue and the vowels.
However, the results of the zero-shot setting revealed that they have difficulty determining the tongue positions from a given image, as mentioned in Section~\ref{sec:clip_lm_tongue_positions}.
The fine-tuned models' results also demonstrate that learning these relationships through objective observation is challenging.
It can be posited that LVLMs lack an understanding of tongue positions, as can be inferred from empirical knowledge and objective observation.

We have also discussed in Sections \ref{sec:comparison_dataset}, \ref{sec:clip_lm_tongue_positions}, and \ref{sec:do_lms_consider_tongue_position} that some LVLMs can consider relative positions to predict vowels.
These findings reveal that LVLMs predict vowels with respect to other pairs of tongue positions and vowels, whereas predicting unseen vowels is challenging, as shown in the results of the one-shot setting. We can conclude that LVLMs, to a certain extent, associate tongue positions with the concept of vowel articulation.
Section \ref{sec:cot} finds that explicit textual information using Chain-of-Thought inhibits inference using visual information.

In conclusion, our findings indicate whether LVLMs can comprehend the vowel articulation that linguists have long sought to decode.
While LVLMs faced challenges in our experiments, the performance of LVLMs can be improved, like that of linguists, when given examples.
We not only hope these findings will apply to large-scale linguistic analysis, speech synthesis, and educational fields, but also wish for further research on languages.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S8
\section{Ethical Considerations}

%%%%%
\paragraph{License of Source Dataset}
In this study, we created our dataset using the Real-time MRI Articulatory Movement Database (rtMRIDB) \cite{maekawaRealtimeMRIArticulatory2022}.
This dataset is licensed only for research purposes.
Since we have been permitted to use this dataset by the institution providing it, there are no licensing issues.

%%%%%
\paragraph{Identifying Information and Offensive Content}
Our datasets were created from the Real-time MRI Articulatory Movement Database.
We confirm that the original dataset contained no personal information or offensive content, and thus none appears in our dataset.
We also confirm that no inappropriate content is included in our dataset.


\acknowledgment
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
In this study, we used ``Real-time MRI Articulatory Movement Database - Version 1 (rtMRIDB)'' developed by the National Institute for Japanese Language and Linguistics and provided by the Speech Resources Consortium, National Institute of Informatics.
We thank Chihiro Taguchi and the anonymous reviewers for their valuable comments and suggestions.
This work was supported by JSPS KAKENHI Grant Number JP23H03458.


%%%bib
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{04refs.bib}


%%%bio
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}

\bioauthor[:]{Haruki Sakajo}{%
Haruki Sakajo received his Bachelor of Arts in linguistics from Kyoto University in 2024. He is currently a graduate student at the Nara Institute of Science and Technology (NAIST). His research interests include computational linguistics and natural language processing.
}

\bioauthor[:]{Yusuke Sakai}{%
Yusuke Sakai received his B.S. degree in Engineering from Tokyo City University, Tokyo, Japan, in 2020, and his M.S. and Ph.D. degrees in Engineering from the Nara Institute of Science and Technology (NAIST), Ikoma, Japan, in 2022 and 2024, respectively. He is currently an Assistant Professor at NAIST. His research interests include natural language processing, artificial intelligence, machine learning, and linguistics.
}

\bioauthor[:]{Hidetaka Kamigaito}{
Hidetaka Kamigaito received his Ph.D. from Tokyo Institute of Technology. After being an assistant professor at the Tokyo Institute of Technology, he is currently an associate professor at Nara Institute of Science and Technology. His current research interests include natural language processing with an emphasis on natural language generation and knowledge graph completion.
}

\bioauthor[:]{Taro Watanabe}{%
Taro Watanabe received his B.E. and M.E. degrees in information science from Kyoto University in 1994 and 1997, respectively, and obtained an M.S. degree in Language and Information Technologies from the School of Computer Science, Carnegie Mellon University in 2000.
In 2004, he received a Ph.D. in informatics from Kyoto University.
After working as a researcher at ATR, NTT, NICT, and as a software engineer at Google, he was appointed a professor at the Nara Institute of Science and Technology starting in 2020.
His research interests include natural language processing, machine learning, language modeling and machine translation.
}

\end{biography}

\biodate   %%%% Following settings will be edited by editors.


\end{document}

