\documentclass[english]{jnlp_3.8}
\usepackage{jnlpbbl_2.2}
\usepackage[dvipdfmx]{graphicx}
\usepackage{amsmath}
\usepackage{amssymb}
\usepackage{url}
\usepackage{array}
\allowdisplaybreaks

\usepackage{hangcaption_jnlp}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%%%% author's definition

\newcommand\blfootnote[1]{%
  \begingroup
  \renewcommand\thefootnote{}\footnote{#1}%
  \addtocounter{footnote}{-1}%
  \endgroup
}

%%%%%\usepackage{caption}
\usepackage{booktabs}
%%%%%\usepackage{amsthm}
%%%%%\usepackage{mathrsfs}
\usepackage{bm}
\usepackage{multirow}
%%%%%\usepackage{newfloat}
%%%%%\usepackage{listings}
%%%%%\usepackage{footmisc}
%%%%%\usepackage{mathabx}
%%%%%\usepackage{footnote}
%%%%%\makesavenoteenv{tabular}
%%%%%\makesavenoteenv{table}
%%%%%\usepackage{hangcaption_jnlp}

%%%%%\setcounter{topnumber}{3}            % ページの上部に含める図表の数
%%%%%\def\textfraction{.01}                % ページ内の文章領域の最小の割合
%%%%%\setcounter{dbltopnumber}{2}         % ↑の二段組 version
%%%%%%%%%%\def\topfraction{.99}                 % ページの上部で図の占める割合の上限
%%%%%\def\dbltopfraction{.99}              % ↑の二段組 version
%%%%%\def\floatpagefraction{.99}           % 図表のみのページで図表の占める割合の下限
%%%%%\def\dblfloatpagefraction{.99}        % ↑の二段組 version



%%%%%\lstdefinelanguage{json}{
%%%%%    basicstyle=\small\ttfamily,
%%%%%    numbers=none,
%%%%%}


\usepackage{footnote}
\makesavenoteenv{tabular}
\makesavenoteenv{table}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%

\Volume{32}
\Number{2}
\Month{June}
\Year{2025}
%%%%\Issuetitle{「　　　　」特集号}


\received{2024}{11}{1}
\revised{2025}{2}{8}
\accepted{2025}{3}{18}



\setcounter{page}{572}



%%%%   Title, Author, etc.            %%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\esubject{General Paper}
\etitle{Data-to-Text Generation for Esports Game Commentary of Multiplayer Strategy Game}
\eauthor{Zihan Wang\affiref{Author_1} \and Naoki Yoshinaga\affiref{Author_2}} 
\eabstract{%
Esports, a sports competition on video games, has become one of the most important sporting events. Despite the large accumulation of esports play logs, only a small portion are accompanied by text commentaries that help the audience retrieve and understand the plays. In this study, we introduce the task of generating commentaries from esports game's data records. We begin by building large-scale esports data-to-text datasets that pair structured data records with textual commentaries from a popular esports game, League of Legends. We then explore several generation models to produce game commentaries from structured data records while also examining the impact of pre-trained language models. To assess the generated commentaries, we designed evaluation metrics that focused on the unique characteristics of esports data, such as evaluating strategic depth. The experimental results of the data-to-text generation using our dataset revealed the remaining challenges of this novel task.
}
\ekeywords{Data-to-text, Text Generation, Esports, Pre-trained Language Model}

\headauthor{Wang and Yoshinaga}
\headtitle{Data-to-Text Generation for Esports Game Commentary}


%%%%     Affiliation     %%%%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\affilabel{Author_1}{}{The University of Tokyo}
\affilabel{Author_2}{}{Institute of Industrial Science, The University of Tokyo}

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{document}

\maketitle
\blfootnote{Part of this study is presented at the NAACL 2024 Student Research Workshop \cite{wang2024commentary}. This article details the dataset and task settings and reports the evaluation results of LSTM and a hierarchical model, and analyzes the system outputs to address the challenges of leveraging long game histories and integrating visual information.}
\blfootnote{Source code and related information available at \url{https://github.com/ArnoZWang/esports-data-to-text}}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S1
\section{Introduction}
Esports, a sports competition using video games, has surged in popularity, now attracting larger audiences than ever before \cite{hamari2017esports,reitman2020esports}. Despite its growing popularity, individual gameplays accompany a few metadata such as player names, which prevents the esports audience from finding games with strategies of interest and understanding the intention of skillful actions. Human-written game commentaries can help audiences retrieve games using a natural language query, and thus
better understand player actions \cite{lavelle2010critical}. However, it is costly for human experts to provide individual games with play-by-play commentaries. Consequently, only a small fraction of esports games with play logs are well accompanied by textual commentaries.

To enhance the audience's experience of watching sports games, data-to-text generation technology can be used to generate game commentaries from structured data records. In the literature, data-to-text generation has been applied for summary generation from the box and line scores of basketball games \cite{wiseman2017challenges} and play-by-play commentary generation for board games \cite{modgil2013added,kameko2015learning}. Compared with basketball data, esports data contain more detailed game records and play-by-play commentaries. Compared with board game data, esports data are usually not turn-based and emphasize real-time moves. In conclusion, there is a lack of research that considers the characteristics of esports data, particularly on the most popular types of multiplayer strategy games.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F1
\begin{figure}[b]
\begin{center}
\includegraphics{32-2ia6f01.eps}
\end{center}
\hangcaption{Game commentary based on one-minute data records (a series of events in JSON format). The screenshot is to help interpret the input and is not a part of the input.}
\label{fig:esports_data}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



In this study, we introduce the task of generating game commentaries from structured data records (Figure \ref{fig:esports_data}) for one of the most popular esports games, the League of Legends (LoL) \cite{webb2019more,ferrari2013generative}. This task aims to investigate data-to-text generation for esports commentary on multiplayer strategy games. Broadly, the overall workflow for performing this new task includes three main processes. First, we build a large-scale data-to-text generation dataset comprising commentaries 
on esports games and the corresponding structured data records;
we then use several encoder--decoder baselines, along with Transformer-based \cite{vaswani2017attention} pre-trained models \cite{touvron2023llama} to tackle the task and set the evaluation metrics of esports data-to-commentary generation.

We evaluate the performance of esports data-to-commentary generation on our proposed datasets using metrics regarding the unique characteristics of esports data. We then discuss the main challenges of this task that need to be addressed in the future.

The contributions of this study are as follows:
%%%%
\begin{itemize}
\item We set up a task of data-to-commentary generation for one of the most popular esports games, League of Legends.
\item We have collected structured data records and game commentaries to build large-scale esports datasets to facilitate the research on this task; we have released the scripts and related information to construct these datasets (\S \ref{sec:datasets}).\footnote{\url{https://github.com/ArnoZWang/esports-data-to-text}, last accessed January 31, 2025.\label{fn:dataset}}
\item On the constructed datasets, we introduced several encoder--decoder baselines and presented a hierarchical architecture to tackle the esports data-to-commentary generation task; we also investigated the utility of the pre-trained language models (\S \ref{sec:generation}).
\item We designed evaluation criteria tailored for esports data-to-commentary generation, reflecting the unique characteristics of esports games and purposes of game commentaries, and we evaluated the introduced models on the constructed dataset (\S \ref{sec:experiments}).
\item On the basis of the evaluation results, we discussed the remaining challenges of this task and proposed potential directions for future research (\S \ref{sec:analysis}).
\end{itemize}


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S2
\section{Related Work}
\label{sec:rel}
In this section, we review existing data-to-text tasks for sports, board, and video games. We detail the inputs and outputs of these tasks to highlight their characteristics.

%%%%%
\paragraph{Summary generation for sports game records}
Several studies have focused on generating textual summaries of physical sports games primarily from score records and other game statistics \cite{wiseman2017challenges,van2017pass,dou2018data2text,van2018evaluating,taniguchi2019generating,puduppully2019data,rebuffel2020hierarchical,hu-etal-2024-sportsmetrics}. These studies typically concentrate on sports such as basketball, soccer, and American football, where the recorded data include only a few key metrics, such as scores, player numbers, and team names. In contrast, our study addresses esports, specifically within the LoL domain concerning multiplayer strategy games, where game data encompass a far richer array of detailed in-game events and player actions. This increase in data granularity results in a much longer commentary per game in our LoL dataset compared with traditional physical sports records, which challenges the generation model to understand individual actions in the games.

%%%%
\paragraph{Play-by-play commentary generation for board games}
Some studies have addressed the generation of play-by-play commentaries that incorporate grounded move expressions derived from chess and shogi (Japanese chess) games \cite{modgil2013added,kameko2015learning,jhamtani2018learning}. For both esports and board games, it is possible to reconstruct the entire gameplay directly from data records. Nevertheless, board games are turn-based games in which two players perform their actions sequentially in alternating turns, whereas esports games are usually not turn-based. In esports games, multiple players can simultaneously perform actions. These differences create unique challenges for automated commentary generation in esports, necessitating a model or system that can interpret simultaneous actions.

%%%%
\paragraph{Commentary generation from esports game videos}
Several studies have aimed at producing textual summaries and commentaries from game videos  \cite{khan2015video,pasunuru2018game,tanaka2021lol,zhang2024game}. In particular, regarding racing games, a study \cite{ishigaki2021generating} generated live commentary from game video data using structured telemetry data, mostly on the numerical values of the player's car and game progress. Although this study utilizes game data records, they provide only partial information on gameplay and are meant to supplement the visual data. Within the multiplayer strategic sports genre \cite{zhang2022moba,wei2022honor}, our task focuses on the LoL, which is one of the most popular sports games and serves as a representative game of this genre \cite{ferrari2013generative,webb2019more}. Our goal is to generate commentaries from comprehensive structured data records with detailed descriptions of games and more strategic content.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3
\section{Esports Data-to-text Datasets}
\label{sec:datasets}
We have constructed and released large-scale data-to-text datasets for one of the most popular esports games, the LoL, which is also a demonstration sports event in the 2018 and 2022 Asian Games because of its popularity \cite{hallmann2018esports,jenny2017virtual}. A large-scale dataset is the core building block for assessing the feasibility of the data-to-text technology for tasks. Therefore, in this study, we built two datasets, LoL19 (core) and LoL19-21 (extended), from all games in the highest-level tournament of the 2019 and 2019 to 2021 Season World Championship of LoL, respectively.

In this section, we first introduce the basics of the target game, LoL, and then explain the methods used to collect structured data records and textual commentaries on LoL\@. We also explain several data preprocessing methods used to improve the collected datasets.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.1
\subsection{Basics of League of Legends}
In this study, we select LoL\footnote{All references to LoL in this paper refer to its main game mode, Summoner's Rift.} as our research target because of its popularity and representative role as a multiplayer strategy game in esports. In the LoL games, each player controls one game character known as ``champion'' with unique abilities that will improve during the game progress and contribute to the team's overall strategy \cite{cannizzo2015towards}. Two teams, represented as ``blue'' and ``red,'' compete in one game map (Figure \ref{fig:map}), each of which team comprises five players. The goal of both teams is to destroy the opponent's base while protecting their own. As the game progresses, the champions can beat enemy champions, defeat non-player units known as ``monsters,'' and destroy buildings to earn resources. They could use their resources to improve their abilities by purchasing items and upgrading their skills. One game typically lasts from 15 min to over an hour \cite{roelofs2024fog}.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F2
\begin{figure}[b]
\begin{center}
\includegraphics{32-2ia6f02.eps}
\end{center}
\hangcaption{Simplified representation of LoL game map \protect\cite{mapofmoba}, specifically corresponding to the bottom right portion of the screenshot shown in Figure \ref{fig:esports_data}. On this game map, two teams of five players each compete to destroy the opponent's base while defending their own. The game map contains several key regions, including two bases, three lanes (top, middle, and bottom), and four jungle areas. The blue and red circles represent buildings.}
\label{fig:map}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


Compared with physical sports (\textit{e.g.}, basketball and soccer) and board games (\textit{e.g.}, chess and shogi), LoL is complicated in terms of real-time game actions, special rules for digital games, and data record size per game. These factors are the main obstacles to data-to-text generation.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.2
\subsection{Data Extraction and Preprocessing}
To build the core dataset named LoL19, we targeted the games of the highest-level tournament in the 2019 World Championship of the LoL\@. We collected the structured data records of the gameplays from the LoL official API site\footnote{\url{https://developer.riotgames.com/}, last accessed January 31, 2025.\label{fn:repeat}} as input and extracted subtitles of YouTube videos on the gameplays as output commentaries. These data were strictly paired with the game IDs provided by the game's Match History site.\footnote{\url{https://lol.fandom.com/wiki/2019_Season_World_Championship/Match_History}, last accessed January 31, 2025.} Subsequently, we introduced a method to process the collected data and build large-scale data-to-commentary generation datasets.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T1
\begin{table}[b]
\input{06table01.tex}
\hangcaption{Statistics (proportions) of all ten types of game events in our LoL datasets.}
\label{table:event}
{\vskip -17pt}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.2.1
\subsubsection{Retrieving data records on gameplays}
In LoL games, every move made by a player is recorded, and the records are available on the LoL official API site.{\kern0pt}$^{3}$
From the LoL data, we can strictly restore the entire game from structured data records, which is impossible in physical sports games such as basketball and soccer. However, a complete dataset contains redundant information, and its large volume provides a significant amount of details, placing a heavy burden on both storage and subsequent processing.

Therefore, we select another data type provided by the official API, ``event-based data frame.'' In these data, individual gameplays are recorded based on associated events. Each event is defined by the official API as an update of certain game status and includes the key named ``\texttt{type},'' which denotes a type of event. Different types of events have various sets of keys, and Table \ref{table:event} lists all ten event types with their proportions, numbers of keys, and explanations. For instance, the ``\texttt{WARD\_PLACED}'' events involve information about ``wardType,'' whereas the ``\texttt{ITEM\_PURCHASED}'' events do not have this key. The event-based data frames are stored in JSON format, as shown in the example in Figure \ref{fig:esports_data}.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.2.2
\subsubsection{Retrieving textual commentaries}
We collected YouTube subtitles from the LoL contest videos as the output of this task.
The subtitles were split into sentences (precisely, utterances) using line breaks provided by YouTube's automatic speech recognition (ASR) as clues. We then randomly selected 200 examples obtained with the following data formatting and manually confirmed their qualities. The resulting word error rate was 6.8, which is comparable to human performance on common ASR datasets,\footnote{\url{https://github.com/syhw/wer_are_we}, last accessed January 31, 2025.} to confirm that the retrieved data were clean for evaluation. We also found that the number of unique token types exceeds 1.3 million, highlighting the high diversity of the collected commentaries.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.2.3
\subsubsection{Data splitting}
The average commentary length for a single LoL game exceeds 10 K words, which is considerably longer than the outputs of existing data-to-text datasets \cite{wiseman2017challenges,jhamtani2018learning}. This extensive length makes it impractical to use a common Transformer architecture \cite{vaswani2017attention} for this task.

To avoid excessively long inputs and outputs, we decompose the obtained structured data and commentary pairs into smaller pieces. First, we divided the event sequence based on one-minute intervals of gameplay. Then, we aligned and segmented the commentary by matching each entry's timestamp to the corresponding event intervals.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.2.4
\subsubsection{Data formatting}
We address the format difference between 
JSON input
and natural language text to make common encoder--decoder models \cite{sutskever2014sequence,vinyals2016order} applicable. Specifically, we linearize the structured input. For each key-value pair in the top-level list of each event in the JSON format, we recursively concatenate the value and the key with a delimiter ``\texttt{$\mid$}'' while inserting a space between individual key-value pairs, with a special token ``\texttt{<ent>$\mid$<ent>}'' inserted at the beginning of the event. For instance, following this procedure, the ``\texttt{WARD\_PLACED}'' event in the JSON format (Figure \ref{fig:esports_data}) is linearized into the following sequence:
%%%%
\begin{quote}
\texttt{<ent>$\mid$<ent> WARD\_PLACED$\mid$type} \texttt{905433$\mid$timestamp} \texttt{YELLOWTRINKET$\mid$wardType} \texttt{6$\mid$creatorId}
\end{quote}
%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.2.5
\subsubsection{Statistics and comparison with other common datasets}
Table \ref{tab:dataset} lists the statistics of the resulting dataset, LoL19, and other common datasets for data-to-text and game commentary generation (\S \ref{sec:rel}), such as LoL-V2T \cite{tanaka2021lol}, Rotowire \cite{wiseman2017challenges}, GameKnot \cite{jhamtani2018learning}, and Dota2-Commentary \cite{zhang2022moba} We also collected data from all games during the 2020 and 2021 Season World Championship of the LoL to extend the LoL19 dataset (LoL19-21). Our datasets have a comparable number of examples to LoL-V2T (esports video-to-text) and RotoWire (basketball data-to-text), and a comparable number of input and output tokens to RotoWire. In the case of GameKnot (chess data-to-commentary), chess games typically involve several turns of player actions, resulting in a much higher number of examples in the dataset, but fewer tokens per input and output. Meanwhile, the unique event-based data frame of esports data also sets it apart from other common datasets, leading to increased complexity.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T2
\begin{table}[b]
{\vskip -5pt}
\input{06table02.tex}
\hangcaption{Statistics of our esports data-to-text datasets and common datasets for similar tasks. Both input and output are tokenized using spaces as delimiters except for the outputs of Dota2-Commentary that contain Chinese text, where we regard each character as a token. Because the task input comprises structured data records (not normal natural language text), we also include the number of characters to better highlight differences in length.}
\label{tab:dataset}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



We also compared our dataset with the Dota2-Commentary \cite{zhang2022moba}, a Chinese commentary generation dataset for Dota 2, which is another multiplayer strategy game. Whereas Dota2-Commentary provides short commentaries for given structured data and videos of single events, our datasets originate from live commentaries given by color commentators who watch gameplays, which results in more insightful and complex commentaries that are aware of past and future contexts.
We have already released scripts{\kern0pt}$^{1}$ for data collection and processing.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S3.3
\subsection{Task Settings and Notations}

In this section, we define the task of esports data-to-text generation. The input of our task was structured data records of short gameplays, and the output was a corresponding textual commentary, defined as follows:

%%%%%
\paragraph{Event}
An event $\bm{e}_i$ is a sequence of key-value pairs, $r_j^{(i)}$, namely, $\bm{e}_i = \{r_1^{(i)}, r_2^{(i)}, \ldots, r_{N_i}^{(i)}\}$, where key-value pairs $r_j^{(i)}$ comprise key $k_j^{(i)}$ and value $v_j^{(i)}$ and $N_i$ is the number of key-value pairs at the top-level list in event $\bm{e}_i$. To indicate multiple key-value pairs belonging to the same event, an additional superscript is added.

%%%%%
\paragraph{Input}
An input to our data-to-commentary task, $\bm{x}_j$, is a sequence of linearized events $\bm{e}_k^{(j)}$, namely, $\bm{x}_j = \{\bm{e}_1^{(j)}, \bm{e}_2^{(j)}, \ldots, \bm{e}_{M_j}^{(j)}\}$, where $M_j$ is the number of events in input $\bm{x}_j$. The events in input $\bm{x}_j$ are sorted in ascending order of their value for the key ``timestamp'' of $\bm{e}_k^{(j)}$. An additional superscript is added to indicate multiple events belonging to the same input. In what follows, the key-value pairs belonging to the same event $\bm{e}_{k}^{(j)}$ for the same input $\bm{x}_j$ are described as $\bm{r}_j^{(i,k)} = \{r_1^{(i,k)}, r_2^{(i,k)}, \ldots, r_{N_{j,k}}^{(j,k)}\}$ where $N_{j,k}$ refers to the number of key-value pairs at the top-level list in event $\bm{e}_k^{(j)}$ for input $\bm{x}_j$.

%%%%%
\paragraph{Output}
The output $\bm{y}_j$ is a textual commentary, which is denoted by a sequence of tokens $c_l^{(j)}$, namely, $\bm{y}_j = \{c_1^{(j)}, c_2^{(j)}, \ldots, c_{T_j}^{(j)}\}$, where $T_j$ denotes the number of tokens in $\bm{y}_j$.
  We refer to the sequence from the $p$-th to the $q$-th token of commentary $\bm{y}_j$ as $c_{p:q}^{(j)}$. Commentary $y_j$ can also be denoted as $\bm{y}_j=c_{1:T_j}^{(j)}$.

%%%%%
\paragraph{Dataset}
The esports data-to-commentary generation dataset $\mathcal{D}$ comprises a set of pairs of structured game data and commentary $\mathcal{D}=\{(\bm{x}_i, \bm{y}_i)\}_{i=1}^{L}$, where $L$ is the number of examples in the dataset.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4
\section{Esports Data-to-text Generation}
\label{sec:generation}

For the newly constructed LoL19 dataset, we first introduce baseline encoder--decoder models \cite{sutskever2014sequence,vaswani2017attention} for our esports data-to-text generation task. Subsequently, we present a hierarchical encoder-decoder model that exploits the hierarchical structure of the input sequence of events. We also discuss the utility of pre-trained generative models \cite{radford2019language,raffel2020exploring,touvron2023llama} for this task.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.1
\subsection{Encoder--Decoder Baselines}
We used encoder--decoder models \cite{sutskever2014sequence,vaswani2017attention} as baselines to generate game commentary from the given structured data. The encoder takes structured data, which are a sequence of linearized events, and converts them into fixed-length vectors. The decoder transforms the resulting vectors into commentary sequences. In this study, we explored two encoder--decoder models that used either long-short term memory (\textbf{LSTM}) \cite{hochreiter1997long} or \textbf{Transformer} \cite{vaswani2017attention} as baselines.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.2
\subsection{Hierarchical Model}
This section presents a hierarchical model design to enhance standard encoder--decoder baselines. Given the unique characteristics of esports data, we evaluated a comprehensive model architecture that integrates a hierarchical Transformer encoder with an LSTM decoder, following previous studies on data-to-text generation \cite{rebuffel2020hierarchical}.

We can easily observe that the input to this task is composed of events and exhibits an evident two-level structure, where each event is composed of a sequence of key-value pairs. For this data structure, flat encoders cannot distinguish interactions among events and key-value pairs within each event, which can lead to failure to understand the input. To address this limitation, we implemented a hierarchical encoder as a replacement for a flat encoder.


%%%%%%%%%%%%%%%%%%%%%%%%%F3
\begin{figure}[b]
\begin{center}
\includegraphics{32-2ia6f03.eps}
\end{center}
\hangcaption{Overall hierarchical model structure of esports data-to-commentary generation.}
\label{fig:overall_model}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%


Specifically, the implemented model comprises a hierarchical Transformer-based encoder \cite{rebuffel2020hierarchical} and an LSTM-based decoder \cite{hochreiter1997long,wang2016experimental} (\textbf{Hierarchical}). Figure \ref{fig:overall_model} illustrates the model. The hierarchical encoder first employs a low-level encoder that feeds key-value pairs within each event $e_i$ to obtain its representation. Subsequently, a high-level encoder is used that feeds the embedding of the last layer for the last token as an event representation encoded by the low-level encoder. The hierarchical attention mechanism involves learning two sets of attention scores: the first set over events $e_k^{(j)}$ for input $\bm{x}_j$ and the second set over data records $r_{m}^{(j,k)}$ for event $e_k^{(j)}$.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S4.3
\subsection{Pre-trained Language Models}
With the recent growth in pre-trained generative language models \cite{radford2019language,raffel2020exploring}, we investigated their utility in our task. We select a standard generative pre-trained model based on an encoder--decoder Transformer architecture,  \textbf{T5} \cite{raffel2020exploring},\footnote{\url{https://huggingface.co/t5-base}, last accessed January 31, 2025.} to evaluate its performance in our esports data-to-commentary task. Additionally, with numerous studies advocating scaling up the parameters of language models to the billion-parameter level, we also evaluated the models of the \textbf{Llama2} family \cite{touvron2023llama}, which range in scale from 7B to 70B parameters and are based on the decoder-only Transformer architecture.\footnote{\url{https://huggingface.co/meta-llama}, last accessed January 31, 2025.} T5 was pre-trained only on the text data of 34B tokens from the Colossal Clean Crawled Corpus (C4),\footnote{\url{https://www.tensorflow.org/datasets/catalog/c4}, last accessed March 22, 2025. Note that C4 applies heuristics to explicitly filter out the codes.} Llama2 models are pre-trained on 2T tokens of text and code data, which equips the Llama2 models with a higher ability to understand linearized JSON inputs and generate more fluent commentaries than T5.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5
\section{Experiments}
\label{sec:experiments}
In this section, we conduct experiments on esports data-to-text generation using the newly constructed LoL19 dataset. We first list the experimental settings and thereafter introduce a new set of evaluation metrics, including human scoring criteria, for strategic depth evaluation. Based on these metrics, we evaluate and compare the performance of several encoder--decoder baselines and pre-trained language models.

\clearpage%%%%%

%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.1
\subsection{Settings}
\label{sec:settings}

%%%%
\paragraph{Datasets}
The core dataset was used for the evaluation. We first split the games into training, validation, and test sets in a ratio of 8:1:1 according to the chronological order of the individual games, resulting in 2790:350:350 examples. As the core dataset exclusively comprised data from the 2019 Season World Championship, the terminology used within the games, such as player names, was guaranteed to be consistent across the datasets.

%%%%
\paragraph{Models}
We compared models based on a standard encoder--decoder structure, hierarchical encoder, and pre-trained language models (\S \ref{sec:generation}). Both the Transformer and LSTM were implemented using OpenNMT\footnote{\url{https://github.com/OpenNMT/OpenNMT-py}, version 1.0.0, last accessed January 31, 2025.} library. We set the word embedding size to 600, the global attention function to softmax, and model data type of fp32. We set Transformer layers of 6 and a hidden Transformer feed-forward size of 1024. We set LSTM layers of 2 and a hidden RNN of size 600. We also incorporated the copy mechanism \cite{roberti2019copy} into the decoder. For Llama2-based pre-trained models, we use Llama-2-7b, -13b, and -70b.\footnote{\url{https://huggingface.co/collections/meta-llama/llama-2-family-661da1f90a9d678b6f55773b}, last accessed January 31, 2025.}

%%%%
\paragraph{Training}
 For all encoder--decoder baselines, Hierarchical, and T5, we set a decoder dropout of 0.5, training steps of 10,000, and a learning rate of 0.001; the other hyperparameters followed their default settings. We use the Adam optimizer with $\beta 1$ of 0.9 and $\beta 2$ of 0.999. For Llama2-7B and -13B, we applied fine-tuning using QLoRA \cite{dettmers2024qlora} with 4-bit precision. We set the LoRA dropout of 0.1, training steps, and learning rate similar to the Transformer, and the other hyperparameters follow their default settings as in Huggingface Llama2\footnote{\url{https://huggingface.co/docs/Transformers/model_doc/llama2}, last accessed January 31, 2025.} document. We trained the pre-trained models using Python 3.11.5 and PyTorch 2.1.2. However, for OpenNMT-based models, we recommend using older versions considering compatibility, specifically Python 3.6.0 and PyTorch 1.1.0. For Llama2-70B, we applied zero-shot in-context learning (ICL) \cite{floridi2020gpt} without updating the model weights because of the large number of parameters. Specifically, the prompts used in ICL are as follows:
\begin{quote}
\textit{You are an expert of League of Legends esports games. Please read the input data records and describe them in natural language commentary as output. Input: [insert input data here] Output:}
\end{quote}



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.2
\subsection{Evaluation Metrics}
Building upon previous studies on data-to-text tasks, including sports game summaries {\linebreak}\cite{puduppully2019data,rebuffel2020hierarchical}, board game commentaries \cite{jhamtani2018learning}, and racing game commentaries  \cite{ishigaki2021generating}, we adopted several commonly used evaluation metrics to assess text quality. Specifically, we employed \textbf{sacreBLEU} \cite{papineni2002bleu,post2018call}\footnote{\url{https://github.com/mjpost/sacrebleu/releases/tag/v1.4.13}, last accessed January 31, 2025.}, \textbf{text distance} (normalized Damerau-Levenshtein)\footnote{\url{https://github.com/life4/textdistance}, version 4.2.2, last accessed January 31, 2025.} \cite{brill2000improved}, along with \textbf{BERTScore} \cite{bertscore}\footnote{\url{https://pypi.org/project/bert-score}, version 0.3.13, last accessed January 31, 2025.} and \textbf{BARTScore} \cite{NEURIPS2021_e4d2b6e6}.\footnote{\url{https://github.com/neulab/BARTScore}, SHA 248f511cb34ae3753fc81f7d7a945de5bfe33458, last accessed January 31, 2025.} These automatic metrics collectively aim to reflect the quality of the generated results over correctness and fluency evaluations.

Given the unique characteristics of multiplayer strategy esports games, it is important to assess the strategic depth of game commentaries. Thus, the \textbf{strategic depth} evaluation is designed to measure the extent to which the system output provides useful information regarding the players' actions. Although the automatic metrics above can arguably measure the general qualities of the system output, we also want the output to contain strategically relevant content, such as reflecting the players' intentions and the team's arrangement regarding combat.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T3
\begin{table}[b]
{\vskip -2pt}
\input{06table03.tex}
\hangcaption{Scoring criteria of the strategic depth evaluation.}
\label{table:strategic_depth_score}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


As it is difficult to estimate strategic depth automatically, we gathered human scores using criteria tailored for esports commentaries, as detailed in Table \ref{table:strategic_depth_score}. These scores were collected from five graduate students who were human annotators. Each annotator was knowledgeable about the rules of LoL, the content of contest games, and English-language LoL game commentaries. We then calculated the average number of scores.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S5.3
\subsection{Results}
Table \ref{table:result_small} compares the performances of the encoder--decoder baselines and 
pre-trained language models on our LoL19 dataset. Among the encoder--decoder baselines, the LSTM outperformed the Transformer in terms of sacreBLEU, text distance, and strategic depth. However, the Transformer performs better in BERTScore and BARTScore evaluations. In addition, the hierarchical model surpassed the standard Transformer and LSTM across all metrics, highlighting the advantages of its hierarchical architecture. For the pre-trained language models, compared with the standard Transformer, T5 showed improvements in both sacreBLEU and strategic depth scores. Llama2-13B exhibited the best overall performance, which confirms the enhancement resulting from pre-trained models with larger sizes. We also compared the performances of Llama2-7B and -13B without finetuning in an ablation study. For Llama2-70B, although ICL did not result in high text similarity, the generated results exhibited the potential to produce inspiring content.


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%T4
\begin{table}[t]
\input{06table04.tex}
\hangcaption{Experimental results on the LoL19 esports data-to-text generation dataset.}
\label{table:result_small}
\end{table}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%


%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F4
\begin{figure}[b]
\begin{center}
\includegraphics[width=\textwidth]{32-2ia6f04.eps}
\end{center}
\hangcaption{Excerpts of system outputs for an input including an \texttt{ITEM\_PURCHASED} event.}
\label{fig:visual_example_3}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F5
\begin{figure}[b]
\begin{center}
\includegraphics[width=\textwidth]{32-2ia6f05.eps}
\end{center}
\hangcaption{Excerpts of system outputs for an input including an \texttt{BUILDING\_KILL} event.}
\label{fig:visual_example_4}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%F6
\begin{figure}[b]
\begin{center}
\includegraphics[width=\textwidth]{32-2ia6f06.eps}
\end{center}
\hangcaption{Excerpts of system outputs for an input including an \texttt{CHAMPION\_KILL} event.}
\label{fig:visual_example_5}
\end{figure}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%




%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6
\section{Analysis}
\label{sec:analysis}
In this section, we first select several examples of generations to demonstrate the performance of commentary generation and to highlight the difficulty of this task. Next, the remaining issues and challenges associated with this task are discussed.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.1
\subsection{Examples of Generation}
\label{sec:examples}

In the first example (Figure \ref{fig:visual_example_3}), the reference output revisits past events in the game to make comments, because the current period of the game is relatively quiet, without sufficiently informative ongoing events. Some system outputs accurately describe the ongoing event (\textit{e.g.}, ``items being destroyed and purchased'' in the Llama2-70B result). However, they primarily focused on the present moment rather than incorporating a more informative context.

In the second example (Figure \ref{fig:visual_example_4}), although the remaining players gathered in another area of the map, the lone player in the top lane was taking an enemy building down a little while ago. This was an important event because it directly affected the direction of the game. Although most system outputs successfully capture the key event (\textit{e.g.}, ``it's going to go down'' in the Hierarchical result, ``top lane is going to be collected'' in the Llama2-13B result), an effective commentary should go beyond merely describing the present game moment. Instead, they should provide deeper and more helpful insights.

In the third example (Figure \ref{fig:visual_example_5}), the red team successfully defeated an enemy champion and seized the critical opportunity to continue their offensive push. This scenario involves a series of complex player actions that occur within a brief time frame of only a few seconds. The ongoing event is closely linked to adjacent ones. These challenges can lead to significant difficulties in system output, which may either provide limited information or even generate inaccurate content.

In conclusion, the use of a finer-segmented input helps enhance the generation accuracy in these cases. However, this approach risks losing important contextual information, causing the system to omit content related to the game history. Balancing the input size with the contextual depth remains challenging. Meanwhile, the primary focus of this task currently relies on the modalities of structured data records and textual game commentaries, whereas visual inputs, such as screenshots and video clips, can contribute to generation. Integrating these visual elements may enhance the cross-modal generation capabilities.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S6.2
\subsection{Remaining Challenges}
The remaining challenges of this task focus on two main aspects: leveraging long game histories effectively and integrating visual information. The system struggles to link past events with current moments owing to segmented data, and lengthening inputs may introduce noise. Visual elements, such as screenshots can add context-rich details that structured data alone lack, offering a more comprehensive view of the game.

%%%%%
\paragraph{Leveraging long game histories}
Based on the examples and results presented in \S \ref{sec:examples}, we conclude that the main difficulty of this task is in associating past events with current events. Human-written commentaries frequently revisit what happened in the game's past to fill in quieter periods. In contrast, the system output can only describe the present moment in the game, because the original data are already split into finer segments. Although these finer segments have the potential for event-specific generation, they also lead to the loss of context, making it difficult for the system output to produce more insightful content that references game history (past events). A straightforward solution would be to increase the length of the inputs and outputs to capture extended game periods; however, this approach may introduce noise and additional issues for Transformer-based models \cite{dong2023survey,li2025longcontext}. 
In summary, striking a balance between the input length and contextual completeness remains a serious challenge. We need to consider extended contexts, which have been confirmed to be effective in neural machine translation \cite{tiedemann-scherrer-2017-neural,bawden-etal-2018-evaluating,voita-etal-2018-context,sugiyama-yoshinaga-2019-data,voita-etal-2019-context,yu-etal-2020-better,sugiyama-yoshinaga-2021-context,petrick-etal-2023-document}, and summarize or select past contexts to understand the current input, as has been confirmed to be effective in long-term open-domain conversation \cite{xu-etal-2022-beyond,xu-etal-2022-long,bae-etal-2022-keep,takasaki2023effective,nishida2023perplexity}.

%%%%
\paragraph{Integrating visual information}
Another approach to address the long-context issue is to integrate visual data into this task. Visual elements of esports games, such as screenshots and video clips, are the main source for humans (including commentators) to access games. Although this task focuses on data records that directly correspond to visual information (game scenes), these data records can only describe part of the scene. By contrast, the game scene has more details, which can provide more insights and result in a more inspiring output. For instance, in the game scene presented in Figure \ref{fig:visual_example_4}, the input data record can only tell that the main player is taking down a building; however, the screenshot reveals further details such as ``this player is alone in the top lane,'' ``this player is close to an enemy building but far from his teammates,'' and various in-game values (scores, player IDs, and team IDs). Visual information can effectively capture brief moments in the history of a game. Combining data records with multiple screenshots will provide a summary of recent events to reflect past events, enriching the context-aware generation process while avoiding the drawbacks of using data records only. A previous study \cite{tanaka2021lol} explored vision-to-text generation for esports using visual inputs and textual outputs. However, a gap remains in fully leveraging all three relevant modalities of multiplayer strategy esports games\textemdash data records, visual elements, and textual commentaries\textemdash to conduct comprehensive multimodal esports commentary generation.



%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%S7
\section{Conclusions}
This study set up a task for generating game commentaries from structured data of the popular multiplayer strategy game, the League of Legends. We developed large-scale data-to-text generation datasets for a strategic esports game. Next, we examine the evaluation metrics for our task to assess the quality and strategic depth of the system outputs. We then evaluated encoder--decoder baselines, a hierarchical model, and pre-trained models on this task, revealing the challenges of the task, such as associating past events with ongoing events.

Future studies should address several limitations. The unique challenges include i) effectively linking game history, such as relevant past events, to current game events, and ii) integrating visual inputs, mainly screenshots and video clips, to enable cross-modal understanding and enhance the generation of commentaries. Additionally, although real-time commentary generation was not the primary focus of our previous research, our datasets can serve as a valuable resource for related studies with minor data preprocessing, because they originate from real-time contest videos. Therefore, we hope to examine our datasets to facilitate future research in the field of real-time commentary generation.

\section*{Ethics Statement}
In obtaining data for our LoL datasets, we strictly followed the policies of RiotGames API and YouTube. The former is the publisher of LoL game records. The latter supplied the subtitles of LoL contest videos, which we used as game commentaries in this study. Further ethical concerns related to the game content (\textit{e.g.}, video game content rating) refer to ESRB Rating.\footnote{\url{https://www.esrb.org/ratings/32211/league-of-legends/}, last accessed March 22, 2025.} The game is rated as ``Teen,'' indicating it is suitable for players aged 13 and older.



% Start Acknowledgment
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\acknowledgment
This work was partially supported by JST, CREST Grant Number JPMJCR19A4, Japan and JSPS KAKENHI Grant Number JP21H03494.


% Start bibliography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\bibliographystyle{jnlpbbl_1.7}
\bibliography{06refs}


% Start biography
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\begin{biography}
\vspace{-\baselineskip}

\bioauthor[:]{Zihan Wang}{%
He received his M.Sc.\ degree in Computer Science from the University of Edinburgh in 2018 and Master's degree in Information Science and Technology from the University of Tokyo, in 2021. He is currently a Ph.D. candidate at the Graduate School of Information Science and Technology at the University of Tokyo. His research interests include language generation and cross-modal processing.
}

\bioauthor[:]{Naoki Yoshinaga}{%
He received his Ph.D.\ from the University of Tokyo in 2005. He has been JSPS Research Fellow (DC1, PD) from 2002 to 2008, and Associate Professor at the University of Tokyo since 2016. His research interests include computational linguistics and machine learning, in particular efficient and adaptive natural language processing in the wild.
}
\end{biography}

\biodate   %%%% Following settings will be edited by editors.


\end{document}



