@inproceedings{1e8d56491b9044209fe521bb9333f0a1,
title = "UNSUPERVISED WORD SEGMENTATION BASED ON WORD INFLUENCE",
abstract = "Word segmentation task is the cornerstone of text processing. There are 7111 languages worldwide, most of which are low-resource languages. This paper attempts to solve the problem of multilingual unsupervised word segmentation using common points between languages without tagged corpus. We find that words are only a relationship between phrases and non-phrases in each language, and the frequency of their occurrence obeys the normal distribution. Based on the objective law of language and pre-training language model, this paper defines the concept of Word Influence and designs its calculation formula, and loss function. Combined with the fine-tuning word segmentation task, a multilingual unsupervised word segmentation model was proposed. In order to apply to multiple languages, the model{\textquoteright}s key parameters can be learned independently. Its validity and advancement have been proved on Chinese, Japanese, and English data sets. Finally, we discuss the challenges of word segmentation in the pre-trained language model environment.",
keywords = "Multilingual, Pre-trained model, Unsupervised, Word Influence, Word Segmentation",
author = "Ruohao Yan and Huaping Zhang and Wushour Silamu and Askar Hamdulla",
note = "Publisher Copyright: {\textcopyright}2023 IEEE.; 48th IEEE International Conference on Acoustics, Speech and Signal Processing, ICASSP 2023 ; Conference date: 04-06-2023 Through 10-06-2023",
year = "2023",
doi = "10.1109/ICASSP49357.2023.10096718",
language = "English",
series = "ICASSP, IEEE International Conference on Acoustics, Speech and Signal Processing - Proceedings",
publisher = "Institute of Electrical and Electronics Engineers Inc.",
booktitle = "ICASSP 2023 - 2023 IEEE International Conference on Acoustics, Speech and Signal Processing, Proceedings",
address = "United States",
}