Merge pull request #1253 from PyThaiNLP/copilot/increase-test-coverage

Move tltk tests from noauto to extra tier

Author: bact

pyproject.toml

@@ -50,6 +50,7 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Intended Audience :: Developers",
     "Natural Language :: Thai",
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
@@ -186,6 +187,7 @@ extra = [
     "pandas>=0.24",
     "ssg>=0.0.8",
     "symspellpy>=6.7.6",
+    "tltk>=1.10",
 ]
 
 # Full dependencies - pinned where available

tests/compact/testc_tokenize.py

@@ -23,6 +23,7 @@
     SENT_4,
     TEXT_1,
 )
+from ..test_helpers import assert_segment_handles_none_and_empty
 
 
 class SentTokenizeCRFCutTestCaseC(unittest.TestCase):
@@ -79,8 +80,7 @@ def test_subword_tokenize(self):
 
 class WordTokenizeICUTestCaseC(unittest.TestCase):
     def test_icu(self):
-        self.assertEqual(pyicu.segment(None), [])
-        self.assertEqual(pyicu.segment(""), [])
+        assert_segment_handles_none_and_empty(self, pyicu.segment)
         self.assertEqual(
             word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="icu"),
             ["ฉัน", "รัก", "ภาษา", "ไทย", "เพราะ", "ฉัน", "เป็น", "คน", "ไทย"],

tests/compact/testc_util.py

@@ -2,8 +2,7 @@
 # SPDX-FileType: SOURCE
 # SPDX-License-Identifier: Apache-2.0
 
-"""Unit tests for pythainlp.util module.
-"""
+"""Unit tests for pythainlp.util module."""
 
 import unittest
 

tests/core/test_robustness.py

@@ -253,4 +253,3 @@ def test_word_tokenize_with_very_long_strings(self):
                         f"word_tokenize (engine={engine}) failed with "
                         f"very long string (index={i}): {e}"
                     )
-

tests/core/test_tag.py

@@ -6,6 +6,7 @@
 from os import path
 
 from pythainlp.tag import (
+    NER,
     PerceptronTagger,
     perceptron,
     pos_tag,
@@ -88,6 +89,11 @@ def test_pos_tag(self):
             ],
         )
 
+    def test_NER_error_handling(self):
+        # Test error handling for invalid engine/corpus combination
+        with self.assertRaises(ValueError):
+            NER(engine="thainer", corpus="cat")
+
 
 class PerceptronTaggerTestCase(unittest.TestCase):
     """Test pythainlp.tag.PerceptronTagger

tests/core/test_tokenize.py

@@ -11,6 +11,7 @@
     longest,
     multi_cut,
     newmm,
+    paragraph_tokenize,
     sent_tokenize,
     subword_tokenize,
     syllable_tokenize,
@@ -22,6 +23,8 @@
 )
 from pythainlp.util import dict_trie
 
+from ..test_helpers import assert_segment_handles_none_and_empty
+
 TEXT_1 = "หมอนทองตากลมหูว์MBK39 :.ฉฺ๐๐๓-#™±"
 TEXT_2 = "ทดสอบ"
 
@@ -231,7 +234,7 @@ def test_word_detokenize(self):
         )
 
     def test_numeric_data_format(self):
-        engines = ["newmm"]
+        engines = ["newmm", "longest"]
 
         for engine in engines:
             self.assertIn(
@@ -257,6 +260,35 @@ def test_numeric_data_format(self):
             self.assertIn("2.5:1", tokens)
             self.assertIn("5:2", tokens)
 
+        # Test join_broken_num parameter (defaults to True)
+        # When True, numeric data should be preserved
+        engine = "longest"
+        self.assertIn(
+            "127.0.0.1",
+            word_tokenize(
+                "ไอพีของคุณคือ 127.0.0.1 ครับ",
+                engine=engine,
+                join_broken_num=True,
+            ),
+        )
+        # When False, numbers may be broken up
+        self.assertNotIn(
+            "127.0.0.1",
+            word_tokenize(
+                "ไอพีของคุณคือ 127.0.0.1 ครับ",
+                engine=engine,
+                join_broken_num=False,
+            ),
+        )
+        self.assertNotIn(
+            "1,234,567.89",
+            word_tokenize(
+                "รางวัลมูลค่า 1,234,567.89 บาท",
+                engine=engine,
+                join_broken_num=False,
+            ),
+        )
+
 
 class TokenizeTestCase(unittest.TestCase):
     def test_Tokenizer(self):
@@ -361,8 +393,7 @@ def test_word_tokenize(self):
             )
 
     def test_etcc(self):
-        self.assertEqual(etcc.segment(None), [])
-        self.assertEqual(etcc.segment(""), [])
+        assert_segment_handles_none_and_empty(self, etcc.segment)
         self.assertIsInstance(etcc.segment("คืนความสุข"), list)
         self.assertEqual(
             etcc.segment("หาเงินเพื่อเรียน"),
@@ -377,8 +408,7 @@ def test_etcc(self):
         )
 
     def test_longest(self):
-        self.assertEqual(longest.segment(None), [])
-        self.assertEqual(longest.segment(""), [])
+        assert_segment_handles_none_and_empty(self, longest.segment)
         self.assertIsInstance(
             longest.segment("กรุงเทพฯมากๆเพราโพาง BKKฯ"), list
         )
@@ -430,8 +460,7 @@ def test_longest_custom_dict(self):
         )
 
     def test_mm(self):
-        self.assertEqual(multi_cut.segment(None), [])
-        self.assertEqual(multi_cut.segment(""), [])
+        assert_segment_handles_none_and_empty(self, multi_cut.segment)
         self.assertIsNotNone(multi_cut.segment("ตัด", dict_trie([""])))
 
         self.assertEqual(word_tokenize("", engine="mm"), [])
@@ -468,8 +497,7 @@ def test_mm(self):
         self.assertEqual(multi_cut.find_all_segment(None), [])
 
     def test_newmm(self):
-        self.assertEqual(newmm.segment(None), [])
-        self.assertEqual(newmm.segment(""), [])
+        assert_segment_handles_none_and_empty(self, newmm.segment)
         self.assertEqual(
             word_tokenize("ฉันรักภาษาไทยเพราะฉันเป็นคนไทย", engine="newmm"),
             ["ฉัน", "รัก", "ภาษาไทย", "เพราะ", "ฉัน", "เป็น", "คนไทย"],
@@ -556,8 +584,7 @@ def test_newmm_dangertext(self):
         )
 
     def test_tcc(self):
-        self.assertEqual(tcc.segment(None), [])
-        self.assertEqual(tcc.segment(""), [])
+        assert_segment_handles_none_and_empty(self, tcc.segment)
         self.assertEqual(
             tcc.segment("ประเทศไทย"), ["ป", "ระ", "เท", "ศ", "ไท", "ย"]
         )
@@ -616,8 +643,7 @@ def test_tcc(self):
         self.assertEqual(tcc.tcc_pos(""), set())
 
     def test_tcc_p(self):
-        self.assertEqual(tcc_p.segment(None), [])
-        self.assertEqual(tcc_p.segment(""), [])
+        assert_segment_handles_none_and_empty(self, tcc_p.segment)
         self.assertEqual(
             tcc_p.segment("ประเทศไทย"), ["ป", "ระ", "เท", "ศ", "ไท", "ย"]
         )
@@ -652,3 +678,12 @@ def test_display_cell_tokenize(self):
         self.assertEqual(display_cell_tokenize("สวัสดี"), ['ส', 'วั', 'ส', 'ดี'])
         self.assertEqual(display_cell_tokenize("ทดสอบ"), ["ท", "ด", "ส", "อ", "บ"])
         self.assertEqual(display_cell_tokenize("ภาษาไทย"), ["ภ", "า", "ษ", "า", "ไ", "ท", "ย"])
+
+    def test_paragraph_tokenize(self):
+        # Test error handling for invalid engine
+        text = (
+            "(1) บทความนี้ผู้เขียนสังเคราะห์ขึ้นมา"
+            "จากผลงานวิจัยที่เคยทำมาในอดีต"
+        )
+        with self.assertRaises(ValueError):
+            paragraph_tokenize(text, engine="non-existent-engine")

tests/extra/testx_augment.py

@@ -22,13 +22,13 @@ def setUp(self):
         self.text2 = "เราอยู่ที่มหาวิทยาลัยขอนแก่น"
 
     def test_WordNetAug(self):
-        nltk.download('omw-1.4', force=True)  # load wordnet
+        nltk.download("omw-1.4", force=True)  # load wordnet
         wordnetaug = WordNetAug()
         self.assertIsNotNone(wordnetaug.augment(self.text))
         self.assertIsNotNone(wordnetaug.find_synonyms("ผม", pos=None))
         self.assertIsNotNone(wordnetaug.augment(self.text, postag=False))
-        self.assertIsNone(postype2wordnet('n', 'abc'))
-        self.assertIsNotNone(postype2wordnet('NOUN', 'orchid'))
+        self.assertIsNone(postype2wordnet("n", "abc"))
+        self.assertIsNotNone(postype2wordnet("NOUN", "orchid"))
 
     # def test_Thai2fitAug(self):
     #     _aug = Thai2fitAug()

tests/extra/testx_spell.py

@@ -3,7 +3,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # Tests for spell functions that need extra dependencies
-# Note: Tests requiring phunspell/tltk/torch/HuggingFace Hub have been moved to tests.noauto
 
 import unittest
 
@@ -20,7 +19,6 @@
 
 class SpellTestCaseX(unittest.TestCase):
     def test_spell(self):
-        # Tests for symspellpy only (phunspell and tltk moved to noauto)
         result = spell("เน้ร", engine="symspellpy")
         self.assertIsInstance(result, list)
         self.assertGreater(len(result), 0)
@@ -30,16 +28,26 @@ def test_spell(self):
         self.assertGreater(len(result), 0)
 
     def test_word_correct(self):
-        # Tests for symspellpy only (phunspell and wanchanberta moved to noauto)
         result = correct("ทดสอง", engine="symspellpy")
         self.assertIsInstance(result, str)
         self.assertNotEqual(result, "")
 
     def test_spell_sent(self):
-        # Tests for symspellpy only (phunspell moved to noauto)
         self.assertIsNotNone(spell_sent(SENT_TOKS, engine="symspellpy"))
 
     def test_correct_sent(self):
-        # Tests for symspellpy only (phunspell and wanchanberta moved to noauto)
         self.assertIsNotNone(correct_sent(SENT_TOKS, engine="symspellpy"))
         self.assertIsNotNone(symspellpy.correct_sent(SENT_TOKS))
+
+
+class SpellTLTKTestCaseX(unittest.TestCase):
+    """Tests for tltk engine spell checking"""
+
+    def test_spell_tltk(self):
+        result = spell("เน้ร", engine="tltk")
+        self.assertIsInstance(result, list)
+        self.assertGreater(len(result), 0)
+
+        result = spell("เดก", engine="tltk")
+        self.assertIsInstance(result, list)
+        self.assertGreater(len(result), 0)

tests/extra/testx_tag.py

@@ -3,16 +3,14 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # Tests for tag functions that need extra dependencies
-# Note: Tests requiring transformers/tltk have been moved to tests.noautotest
 
 import unittest
 
+from pythainlp.tag import pos_tag, tltk
 from pythainlp.tag.thainer import ThaiNameTagger
 
 
 class TagTestCaseX(unittest.TestCase):
-    # Tests for ThaiNameTagger (doesn't require transformers or tltk)
-    # All tltk and transformers-based tests have been moved to tests.noautotest
 
     def test_thai_name_tagger_1_5(self):
         ner = ThaiNameTagger(version="1.5")
@@ -117,3 +115,39 @@ def test_thai_name_tagger_1_4(self):
             )
         )
 
+
+class TagTLTKTestCaseX(unittest.TestCase):
+    """Tests for tltk engine POS tagging and NER"""
+
+    def test_pos_tag_tltk(self):
+        tokens = ["ผม", "รัก", "คุณ"]
+        self.assertIsNotNone(pos_tag(tokens, engine="tltk"))
+        with self.assertRaises(ValueError):
+            tltk.pos_tag(tokens, corpus="blackboard")
+
+    def test_tltk_ner(self):
+        self.assertEqual(tltk.get_ner(""), [])
+        self.assertIsNotNone(tltk.get_ner("แมวทำอะไรตอนห้าโมงเช้า"))
+        self.assertIsNotNone(tltk.get_ner("แมวทำอะไรตอนห้าโมงเช้า", pos=False))
+        self.assertIsNotNone(
+            tltk.get_ner("พลเอกประยุกธ์ จันทร์โอชา ประกาศในฐานะหัวหน้า")
+        )
+        self.assertIsNotNone(
+            tltk.get_ner(
+                "พลเอกประยุกธ์ จันทร์โอชา ประกาศในฐานะหัวหน้า",
+                tag=True,
+            )
+        )
+        self.assertIsNotNone(
+            tltk.get_ner(
+                """คณะวิทยาศาสตร์ประยุกต์และวิศวกรรมศาสตร์ มหาวิทยาลัยขอนแก่น
+                จังหวัดหนองคาย 43000"""
+            )
+        )
+        self.assertIsNotNone(
+            tltk.get_ner(
+                """คณะวิทยาศาสตร์ประยุกต์และวิศวกรรมศาสตร์ มหาวิทยาลัยขอนแก่น
+                จังหวัดหนองคาย 43000""",
+                tag=True,
+            )
+        )