diff --git a/Jenkinsfile b/Jenkinsfile index 04b51c044..3b4997082 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -29,7 +29,7 @@ pipeline { MR_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/03-12-24-1' JA_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-17-24-1' KO_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-04-25-6' - HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-18-26-0' + HI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-28-26-0' DEFAULT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0' } stages { diff --git a/nemo_text_processing/text_normalization/hi/data/measure/math_equal.tsv b/nemo_text_processing/text_normalization/hi/data/measure/math_equal.tsv new file mode 100644 index 000000000..2738fff86 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/measure/math_equal.tsv @@ -0,0 +1 @@ += बराबर \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/measure/math_operation.tsv b/nemo_text_processing/text_normalization/hi/data/measure/math_operation.tsv new file mode 100644 index 000000000..8fdf49823 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/measure/math_operation.tsv @@ -0,0 +1,9 @@ ++ प्लस +- माइनस +/ बटा +÷ बटा +: बटा +× गुणा +* गुणा +· गुणा += बराबर \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/measure/math_separator.tsv b/nemo_text_processing/text_normalization/hi/data/measure/math_separator.tsv new file mode 100644 index 000000000..2a7601f76 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/measure/math_separator.tsv @@ -0,0 +1 @@ +. दशमलव \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/measure/math_unary.tsv b/nemo_text_processing/text_normalization/hi/data/measure/math_unary.tsv new file mode 100644 index 000000000..ed3e27aa0 --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/measure/math_unary.tsv @@ -0,0 +1,2 @@ ++ प्लस +- माइनस \ No newline at end of file diff --git a/nemo_text_processing/text_normalization/hi/data/whitelist/symbol.tsv b/nemo_text_processing/text_normalization/hi/data/whitelist/symbol.tsv new file mode 100644 index 000000000..ade97847d --- /dev/null +++ b/nemo_text_processing/text_normalization/hi/data/whitelist/symbol.tsv @@ -0,0 +1,5 @@ ++ प्लस +- माइनस += बराबर +< लेस देन +> ग्रेटर देन diff --git a/nemo_text_processing/text_normalization/hi/taggers/measure.py b/nemo_text_processing/text_normalization/hi/taggers/measure.py index 97d4d32ac..6052276d3 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/measure.py +++ b/nemo_text_processing/text_normalization/hi/taggers/measure.py @@ -468,6 +468,31 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser + pynutil.insert("\"") ) + # Math Graph + math_separator = pynini.string_file(get_abs_path("data/measure/math_separator.tsv")) + math_operations = pynini.string_file(get_abs_path("data/measure/math_operation.tsv")) + delimiter = pynini.accep(" ") | pynutil.insert(" ") + math_alpha = capitalized_input_graph(letters_map) + equals = pynini.string_file(get_abs_path("data/measure/math_equal.tsv")) + math_unary = pynini.string_file(get_abs_path("data/measure/math_unary.tsv")) + + optional_unary = pynini.closure(math_unary + delimiter, 0, 1) + + digit_or_zero = digit | zero + fractional_bare = digit_or_zero + pynini.closure(insert_space + digit_or_zero) + + flat_decimal = cardinal_graph + pynutil.insert(" ") + math_separator + pynutil.insert(" ") + fractional_bare + + operand = optional_unary + (cardinal_graph | flat_decimal | math_alpha) + + math_expr = operand + pynini.closure(delimiter + math_operations + delimiter + operand) + + math = math_expr + delimiter + equals + delimiter + math_expr + + graph_math = ( + pynutil.insert('units: "math" cardinal { integer: "') + math + pynutil.insert('" } preserve_order: true') + ) + graph_cardinal_percent = ( pynutil.insert("cardinal { ") + optional_graph_negative @@ -505,6 +530,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst, ordinal: GraphFst, ser | pynutil.add_weight(graph_savva, -0.1) | pynutil.add_weight(graph_sadhe, -0.1) | pynutil.add_weight(graph_paune, -0.5) + | pynutil.add_weight(graph_math, 0.1) | address_graph | structured_address_graph ) diff --git a/nemo_text_processing/text_normalization/hi/taggers/whitelist.py b/nemo_text_processing/text_normalization/hi/taggers/whitelist.py index 93505f9f7..b9e1c4159 100644 --- a/nemo_text_processing/text_normalization/hi/taggers/whitelist.py +++ b/nemo_text_processing/text_normalization/hi/taggers/whitelist.py @@ -46,6 +46,7 @@ def _get_whitelist_graph(input_case, file, keep_punct_add_end: bool = False): return graph graph = _get_whitelist_graph(input_case, get_abs_path("data/whitelist/abbreviations.tsv")) + graph |= _get_whitelist_graph(input_case, get_abs_path("data/whitelist/symbol.tsv")) if deterministic: graph |= graph.optimize() diff --git a/nemo_text_processing/text_normalization/hi/verbalizers/measure.py b/nemo_text_processing/text_normalization/hi/verbalizers/measure.py index cba08057d..e92d4e88a 100644 --- a/nemo_text_processing/text_normalization/hi/verbalizers/measure.py +++ b/nemo_text_processing/text_normalization/hi/verbalizers/measure.py @@ -43,7 +43,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): unit = ( pynutil.delete("units: \"") - + pynini.difference(pynini.closure(NEMO_NOT_QUOTE, 1), pynini.accep("address")) + + pynini.difference(pynini.closure(NEMO_NOT_QUOTE, 1), pynini.union("address", "math")) + pynutil.delete("\"") + delete_space ) @@ -71,6 +71,7 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): graph = (graph_cardinal | graph_decimal) + delete_space + insert_space + unit preserve_order = pynutil.delete("preserve_order:") + delete_space + pynutil.delete("true") + delete_space + address = ( pynutil.delete("units: \"address\" ") + delete_space @@ -79,7 +80,25 @@ def __init__(self, cardinal: GraphFst, decimal: GraphFst): + pynini.closure(preserve_order) ) - graph |= address + math_cardinal = ( + pynutil.delete("cardinal {") + + delete_space + + pynutil.delete("integer: \"") + + pynini.closure(NEMO_NOT_QUOTE, 1) + + pynutil.delete("\"") + + delete_space + + pynutil.delete("}") + ) + + math_graph = ( + pynutil.delete("units: \"math\" ") + + delete_space + + math_cardinal + + delete_space + + pynini.closure(preserve_order) + ) + + graph |= address | math_graph self.decimal = graph_decimal delete_tokens = self.delete_tokens(graph) diff --git a/tests/nemo_text_processing/hi/data_text_normalization/test_cases_math.txt b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_math.txt new file mode 100644 index 000000000..b82bebdaa --- /dev/null +++ b/tests/nemo_text_processing/hi/data_text_normalization/test_cases_math.txt @@ -0,0 +1,47 @@ +1-2=5~एक माइनस दो बराबर पाँच +1- 2 = 5~एक माइनस दो बराबर पाँच +y=x +1~वाई बराबर एक्स प्लस एक +x +1 = y~एक्स प्लस एक बराबर वाई +१-२=५~एक माइनस दो बराबर पाँच +१- २ = ५~एक माइनस दो बराबर पाँच +y=x +१~वाई बराबर एक्स प्लस एक +x +१ = y~एक्स प्लस एक बराबर वाई +२ + २ = ४~दो प्लस दो बराबर चार +७ + ३ = १०~सात प्लस तीन बराबर दस +१५ + ५ = २०~पंद्रह प्लस पाँच बराबर बीस +५ - ३ = २~पाँच माइनस तीन बराबर दो +२० - ८ = १२~बीस माइनस आठ बराबर बारह +२ * ३ = ६~दो गुणा तीन बराबर छह +४ * ५ = २०~चार गुणा पाँच बराबर बीस +९ * ३ = २७~नौ गुणा तीन बराबर सत्ताईस +६ / २ = ३~छह बटा दो बराबर तीन +१०० / ४ = २५~एक सौ बटा चार बराबर पच्चीस +२ + ३ + ४ = ९~दो प्लस तीन प्लस चार बराबर नौ +१० - २ - ३ = ५~दस माइनस दो माइनस तीन बराबर पाँच +४ = २ + २~चार बराबर दो प्लस दो +100 + 50 = 150~एक सौ प्लस पचास बराबर एक सौ पचास +1-2+5/3*4=5.66666666667~एक माइनस दो प्लस पाँच बटा तीन गुणा चार बराबर पाँच दशमलव छह छह छह छह छह छह छह छह छह छह सात +1-2=5~एक माइनस दो बराबर पाँच +1- 2 = 5~एक माइनस दो बराबर पाँच +y=x +1~वाई बराबर एक्स प्लस एक +२ = २~दो बराबर दो +२=२~दो बराबर दो +x = 2~एक्स बराबर दो +x=2~एक्स बराबर दो +y = x~वाई बराबर एक्स +-1 + 2 = 1~माइनस एक प्लस दो बराबर एक +-1+2=1~माइनस एक प्लस दो बराबर एक +1 + -2 = -1~एक प्लस माइनस दो बराबर माइनस एक +1 + - 2 = - 1~एक प्लस माइनस दो बराबर माइनस एक ++1 + +2 = +3~प्लस एक प्लस प्लस दो बराबर प्लस तीन +-10 = -10~माइनस दस बराबर माइनस दस +-x = 5~माइनस एक्स बराबर पाँच +y = -x + 1~वाई बराबर माइनस एक्स प्लस एक +-y = -x + -1~माइनस वाई बराबर माइनस एक्स प्लस माइनस एक +10 - 5 = 5~दस माइनस पाँच बराबर पाँच +100 + 200 = 300~एक सौ प्लस दो सौ बराबर तीन सौ +10 + 20 + 30 = 60~दस प्लस बीस प्लस तीस बराबर साठ +x + y + z = 0~एक्स प्लस वाई प्लस ज़ेड बराबर शून्य +-2 + -2 = -4~माइनस दो प्लस माइनस दो बराबर माइनस चार +x + -2 = 10~एक्स प्लस माइनस दो बराबर दस +२ + -२ = ०~दो प्लस माइनस दो बराबर शून्य \ No newline at end of file diff --git a/tests/nemo_text_processing/hi/test_math.py b/tests/nemo_text_processing/hi/test_math.py new file mode 100644 index 000000000..27886121f --- /dev/null +++ b/tests/nemo_text_processing/hi/test_math.py @@ -0,0 +1,33 @@ +# Copyright (c) 2026, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import pytest +from parameterized import parameterized + +from nemo_text_processing.text_normalization.normalize import Normalizer + +from ..utils import CACHE_DIR, parse_test_case_file + + +class TestMath: + normalizer = Normalizer( + input_case='cased', lang='hi', cache_dir=CACHE_DIR, overwrite_cache=False, post_process=False + ) + + @parameterized.expand(parse_test_case_file('hi/data_text_normalization/test_cases_math.txt')) + @pytest.mark.run_only_on('CPU') + @pytest.mark.unit + def test_norm(self, test_input, expected): + pred = self.normalizer.normalize(test_input, verbose=False) + assert pred.strip() == expected.strip() diff --git a/tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh b/tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh index 74e1cf9c9..66f5d3d7c 100644 --- a/tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh +++ b/tests/nemo_text_processing/hi/test_sparrowhawk_normalization.sh @@ -112,10 +112,10 @@ testTNAddress() { runtest $input } -#testTNMath() { -# input=$PROJECT_DIR/en/data_text_normalization/test_cases_math.txt -# runtest $input -#} +testTNMath() { + input=$PROJECT_DIR/hi/data_text_normalization/test_cases_math.txt + runtest $input +} # Load shUnit2 . $PROJECT_DIR/../shunit2/shunit2