diff --git a/Jenkinsfile b/Jenkinsfile index 04b51c044..2b01d4536 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' - HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-18-26-0' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-22-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { diff --git a/nemo_text_processing/text_normalization/hi/data/roman/roman_ambiguous.tsv b/nemo_text_processing/text_normalization/hi/data/roman/roman_ambiguous.tsv new file mode 100644 index 000000000..63f5803aa --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/roman/roman_ambiguous.tsv @@ -0,0 +1,22 @@ +I I +V V +X X +L L +C C +D D +M M +CD CD +DC DC +MD MD +CM CM +MM MM +MC MC +CC CC +CV CV +XL XL +XXL XXL +XXX XXX +MIX MIX +DIV DIV +LIV LIV +CIV CIV \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/roman/roman_components.tsv b/nemo_text_processing/text_normalization/hi/data/roman/roman_components.tsv new file mode 100644 index 000000000..b6cbb191f --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/roman/roman_components.tsv @@ -0,0 +1,30 @@ +I 1 +II 2 +III 3 +IV 4 +V 5 +VI 6 +VII 7 +VIII 8 +IX 9 +X 1 +XX 2 +XXX 3 +XL 4 +L 5 +LX 6 +LXX 7 +LXXX 8 +XC 9 +C 1 +CC 2 +CCC 3 +CD 4 +D 5 +DC 6 +DCC 7 +DCCC 8 +CM 9 +M 1 +MM 2 +MMM 3 \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/roman/roman_whitelist_context.tsv b/nemo_text_processing/text_normalization/hi/data/roman/roman_whitelist_context.tsv new file mode 100644 index 000000000..f57798815 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/roman/roman_whitelist_context.tsv @@ -0,0 +1,23 @@ +अध्याय अध्याय +भाग भाग +खंड खंड +चरण चरण +टियर टियर +ग्रेड ग्रेड +कक्षा कक्षा +वर्ग वर्ग +प्रकार प्रकार +स्तर स्तर +सदी सदी +शताब्दी शताब्दी +विश्व युद्ध विश्व युद्ध +संस्करण संस्करण +अनुच्छेद अनुच्छेद +परिशिष्ट परिशिष्ट +वार्ड वार्ड +फेज फेज +योजना योजना +राजमार्ग राजमार्ग +कड़ी कड़ी +मिशन मिशन +जहाज़ जहाज़ \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/taggers/roman.py b/nemo_text_processing/text_normalization/hi/taggers/roman.py index ea8d259fe..8b6a349ae 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/roman.py +++ b/nemo_text_processing/text_normalization/hi/taggers/roman.py @@ -32,11 +32,25 @@ class RomanFst(GraphFst): for False multiple transduction are generated (used for audio-based normalization) """ - def __init__(self, deterministic: bool = True): + def __init__(self, cardinal: GraphFst, deterministic: bool = True): super().__init__(name="roman", kind="classify", deterministic=deterministic) - roman_graph = pynini.string_file(get_abs_path("data/roman/roman_to_spoken.tsv")).optimize() - roman_numeral_only = pynini.project(roman_graph, "input").optimize() + components = load_labels(get_abs_path("data/roman/roman_components.tsv")) + ones = pynini.string_map(components[0:9]).optimize() + tens = pynini.string_map(components[9:18]).optimize() + hundreds = pynini.string_map(components[18:27]).optimize() + thousands = pynini.string_map(components[27:30]).optimize() + + zero = pynutil.insert("0") + opt_hundreds = hundreds | zero + opt_tens = tens | zero + opt_ones = ones | zero + + roman_to_arabic = pynini.union( + thousands + opt_hundreds + opt_tens + opt_ones, hundreds + opt_tens + opt_ones, tens + opt_ones, ones + ).optimize() + + roman_to_spoken_fst = pynini.compose(roman_to_arabic, cardinal.graph_without_leading_zeros).optimize() devanagari_chars = pynini.project( pynini.string_file(get_abs_path("data/serial/chars.tsv")), "input" @@ -50,7 +64,39 @@ def __init__(self, deterministic: bool = True): separator = (pynini.accep("-") | pynini.accep(" ")).optimize() - key_before_numeral = ( + whitelist_context = pynini.project( + pynini.string_file(get_abs_path("data/roman/roman_whitelist_context.tsv")), "input" + ).optimize() + + ambiguous_romans = pynini.project( + pynini.string_file(get_abs_path("data/roman/roman_ambiguous.tsv")), "input" + ).optimize() + + # Split the core roman_to_spoken_fst into Safe and Ambiguous paths + safe_roman_inputs = pynini.difference(pynini.project(roman_to_arabic, "input"), ambiguous_romans).optimize() + + safe_roman_to_spoken_fst = (safe_roman_inputs @ roman_to_spoken_fst).optimize() + ambiguous_roman_to_spoken_fst = (ambiguous_romans @ roman_to_spoken_fst).optimize() + + whitelist_word = pynini.project( + pynini.string_file(get_abs_path("data/roman/roman_whitelist_context.tsv")), "input" + ).optimize() + + ambiguous_romans = pynini.project( + pynini.string_file(get_abs_path("data/roman/roman_ambiguous.tsv")), "input" + ).optimize() + + # Phrase matcher that allows any Devanagari words before the whitelist word + whitelist_phrase = (pynini.closure(devanagari_phrase + separator, 0, 1) + whitelist_word).optimize() + + # Split the core roman_to_spoken_fst into Safe and Ambiguous paths + safe_roman_inputs = pynini.difference(pynini.project(roman_to_arabic, "input"), ambiguous_romans).optimize() + + safe_roman_to_spoken_fst = (safe_roman_inputs @ roman_to_spoken_fst).optimize() + ambiguous_roman_to_spoken_fst = (ambiguous_romans @ roman_to_spoken_fst).optimize() + + # Path A: Safe Romans (Uses general Hindi context) + safe_key_before_numeral = ( pynutil.insert("preserve_order: true ") + pynutil.insert('key_cardinal: "') + convert_space(devanagari_phrase) @@ -58,14 +104,14 @@ def __init__(self, deterministic: bool = True): + pynutil.delete(separator) + insert_space + pynutil.insert('integer: "') - + roman_numeral_only + + safe_roman_to_spoken_fst + pynutil.insert('"') ).optimize() - numeral_before_key = ( + safe_numeral_before_key = ( pynutil.insert("preserve_order: true ") + pynutil.insert('integer: "') - + roman_numeral_only + + safe_roman_to_spoken_fst + pynutil.insert('"') + pynutil.delete(separator) + insert_space @@ -74,45 +120,55 @@ def __init__(self, deterministic: bool = True): + pynutil.insert('"') ).optimize() - roman_rows = load_labels(get_abs_path("data/roman/roman_to_spoken.tsv")) - numerals_by_len_desc = sorted((n for n, _ in roman_rows), key=len, reverse=True) + # Path B: Ambiguous Romans (Strictly requires whitelist phrase) + ambiguous_key_before_numeral = ( + pynutil.insert("preserve_order: true ") + + pynutil.insert('key_cardinal: "') + + convert_space(whitelist_phrase) + + pynutil.insert('"') + + pynutil.delete(separator) + + insert_space + + pynutil.insert('integer: "') + + ambiguous_roman_to_spoken_fst + + pynutil.insert('"') + ).optimize() - exception_rows = load_labels(get_abs_path("data/roman/roman_ordinal_exceptions.tsv")) - exception_fused_set = {fused for fused, _ in exception_rows} + ambiguous_numeral_before_key = ( + pynutil.insert("preserve_order: true ") + + pynutil.insert('integer: "') + + ambiguous_roman_to_spoken_fst + + pynutil.insert('"') + + pynutil.delete(separator) + + insert_space + + pynutil.insert('key_cardinal: "') + + convert_space(whitelist_phrase) + + pynutil.insert('"') + ).optimize() - suffix_rows_raw = load_labels(get_abs_path("data/ordinal/suffixes.tsv")) + load_labels( - get_abs_path("data/ordinal/suffixes_map.tsv") - ) + exception_rows = load_labels(get_abs_path("data/roman/roman_ordinal_exceptions.tsv")) exception_graphs = [] for fused, spoken_word in exception_rows: - matched_numeral = next(c for c in numerals_by_len_desc if fused.startswith(c)) exception_graphs.append( - pynutil.insert('integer: "' + matched_numeral + '"') + pynutil.insert('integer: "-"') + insert_space + pynutil.insert('default_ordinal: "' + spoken_word + '"') + pynutil.delete(fused) ) glued_ordinal_exceptions_graph = pynini.union(*exception_graphs).optimize() - regular_row_graphs = [] - for numeral, spoken in roman_rows: - for row in suffix_rows_raw: - - suffix_input = row[0] - suffix_output = row[1] if len(row) > 1 else row[0] - - fused = numeral + suffix_input - if fused in exception_fused_set: - continue - spoken_ordinal = spoken + suffix_output - regular_row_graphs.append( - pynutil.insert('integer: "' + numeral + '"') - + insert_space - + pynutil.insert('default_ordinal: "' + spoken_ordinal + '"') - + pynutil.delete(fused) - ) - glued_ordinal_regular_graph = pynini.union(*regular_row_graphs).optimize() + suffixes_fst = pynini.union( + pynini.string_file(get_abs_path("data/ordinal/suffixes.tsv")), + pynini.string_file(get_abs_path("data/ordinal/suffixes_map.tsv")), + ).optimize() + + glued_ordinal_regular_graph = ( + pynutil.insert('integer: "-"') + + insert_space + + pynutil.insert('default_ordinal: "') + + (roman_to_spoken_fst + suffixes_fst) + + pynutil.insert('"') + ).optimize() roman_glued_ordinal_fields = pynini.union( pynutil.add_weight(glued_ordinal_exceptions_graph, -0.1), @@ -133,6 +189,12 @@ def __init__(self, deterministic: bool = True): ) ).optimize() - graph = pynini.union(key_before_numeral, numeral_before_key, roman_glued_ordinal).optimize() + graph = pynini.union( + safe_key_before_numeral, + safe_numeral_before_key, + ambiguous_key_before_numeral, + ambiguous_numeral_before_key, + roman_glued_ordinal, + ).optimize() self.fst = self.add_tokens(graph).optimize() diff --git a/nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py b/nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py index 04124635e..8e1ec6df2 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py +++ b/nemo_text_processing/text_normalization/hi/taggers/tokenize_and_classify.py @@ -121,7 +121,7 @@ def __init__( word = WordFst(punctuation=punctuation, deterministic=deterministic) word_graph = word.fst - roman = RomanFst(deterministic=deterministic) + roman = RomanFst(cardinal=cardinal, deterministic=deterministic) roman_graph = roman.fst telephone = TelephoneFst() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/roman.py b/nemo_text_processing/text_normalization/hi/verbalizers/roman.py index c28084a77..adb6e61c5 100644 --- a/nemo_text_processing/text_normalization/hi/verbalizers/roman.py +++ b/nemo_text_processing/text_normalization/hi/verbalizers/roman.py @@ -40,13 +40,11 @@ class RomanFst(GraphFst): def __init__(self, deterministic: bool = True): super().__init__(name="roman", kind="verbalize", deterministic=deterministic) - roman_to_spoken = pynini.string_file(get_abs_path("data/roman/roman_to_spoken.tsv")).optimize() - key_cardinal = ( pynutil.delete('key_cardinal: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') ).optimize() - integer = (pynutil.delete('integer: "') + roman_to_spoken + pynutil.delete('"')).optimize() + integer = (pynutil.delete('integer: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"')).optimize() default_ordinal = ( pynutil.delete('default_ordinal: "') + pynini.closure(NEMO_NOT_QUOTE, 1) + pynutil.delete('"') diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt index 340c754ed..3aa88a02d 100644 --- a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_roman.txt @@ -10,7 +10,7 @@ विश्व युद्ध-II~विश्व युद्ध दो प्रथम पंचवर्षीय योजना-I~प्रथम पंचवर्षीय योजना एक राष्ट्रीय राजमार्ग-IV~राष्ट्रीय राजमार्ग चार -रोहिणी आर एस-I~रोहिणी आर एस एक +रोहिणी आर एस-I~रोहिणी आर एस-आई पीएसएलवी सी-IV~पीएसएलवी सी चार ISRO मिशन-III~ISRO मिशन तीन कक्षा XII की परीक्षा~कक्षा बारह की परीक्षा @@ -19,5 +19,35 @@ XIIवीं कक्षा की परीक्षा~बारहवीं अध्याय IV के प्रश्न~अध्याय चार के प्रश्न IVथी कक्षा के विद्यार्थी~चौथी कक्षा के विद्यार्थी XC विद्यार्थी~नब्बे विद्यार्थी -LIII वा गणतंत्र दिन~तिरपन वा गणतंत्र दिन -भाग-XCIX~भाग निन्यानवे \ No newline at end of file +LIII वा गणतंत्र दिन~तिरेपन वा गणतंत्र दिन +भाग-XCIX~भाग निन्यानबे +भाग-CDXLIV~भाग चार सौ चौवालीस +रॉकेट-MMM~रॉकेट तीन हज़ार +अध्याय MCMXCIX~अध्याय एक हज़ार नौ सौ निन्यानबे +MMXXIVवीं वर्षगांठ~दो हज़ार चौबीसवीं वर्षगांठ +MCMXCIX वा स्थापना दिवस~एक हज़ार नौ सौ निन्यानबे वा स्थापना दिवस +MMMCMXCIX सैनिक~तीन हज़ार नौ सौ निन्यानबे सैनिक +हेनरी VIII~हेनरी आठ +सुपर बाउल LVIII~सुपर बाउल अट्ठावन +शताब्दी XXIX~शताब्दी उनतीस +युद्ध III~युद्ध तीन +कक्षा X~कक्षा दस +अध्याय CD~अध्याय चार सौ +टियर I~टियर एक +विश्व युद्ध I~विश्व युद्ध एक +खंड XL~खंड चालीस +फेज IV~फेज चार +चरण XX~चरण बीस +Vवीं सदी~पाँचवीं सदी +Iला भाग~पहला भाग +विटामिन C~विटामिन C +ग्रुप D~ग्रुप D +X रे~X रे +CD प्लेयर~CD प्लेयर +DC साहब~DC साहब +MD साहब~MD साहब +MM लंबाई~MM लंबाई +CC कैमरा~CC कैमरा +शर्ट XL~शर्ट XL +HTML DIV~HTML DIV +डीजे MIX~डीजे MIX \ No newline at end of file