Northern Sámi TN

Jenkinsfile

@@ -22,6 +22,7 @@ pipeline {
     RU_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
     VI_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/10-29-25-0'
     SV_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
+    SE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/06-08-23-0'
     ZH_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/11-13-24-0'
     IT_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/08-22-24-0'
     HE_TN_CACHE='/home/jenkins/TestData/text_norm/ci/grammars/09-24-25-0'
@@ -332,6 +333,24 @@ pipeline {
         }
       }
     }
+    stage('L0: Create SE TN Grammars') {
+      when {
+        anyOf {
+          branch 'main'
+          branch 'staging/**'
+          branch 'staging_*'
+          changeRequest target: 'main'
+        }
+      }
+      failFast true
+      parallel {
+        stage('L0: SE TN grammars') {
+         steps {
+            sh 'CUDA_VISIBLE_DEVICES="" python nemo_text_processing/text_normalization/normalize.py --lang=se --text="100" --cache_dir ${SE_TN_CACHE}'
+          }
+        }
+      }
+    }
     stage('L0: Create JA TN/ITN Grammars') {
       when {
         anyOf {
@@ -520,6 +539,21 @@ pipeline {
 
       }
     }
+
+    stage('L2: SE Sparrowhawk Tests') {
+      when {
+        anyOf {
+          branch 'main'
+          branch 'staging/**'
+          branch 'staging_*'
+          changeRequest target: 'main'
+        }
+      }
+      steps {
+        sh 'CUDA_VISIBLE_DEVICES="" cp -r /workspace/sparrowhawk/documentation/grammars /workspace/sparrowhawk/documentation/grammars_se_tn_grammars_cased && cd tools/text_processing_deployment && bash sh_test.sh --MODE="test_tn_grammars" --INPUT_CASE="cased" --OVERWRITE_CACHE=False --FAR_PATH=${SE_TN_CACHE}/SH_TN --GRAMMARS="tn_grammars" --LANGUAGE="se"'
+        sh 'CUDA_VISIBLE_DEVICES="" cd tests/nemo_text_processing/se && bash test_sparrowhawk_normalization.sh /workspace/sparrowhawk/documentation/grammars_se_tn_grammars_cased `pwd`/..'
+      }
+    }
 
     stage('L2: NeMo text processing') {
       when {

nemo_text_processing/text_normalization/normalize.py

@@ -147,6 +147,9 @@ def __init__(
         elif lang == 'fr':
             from nemo_text_processing.text_normalization.fr.taggers.tokenize_and_classify import ClassifyFst
             from nemo_text_processing.text_normalization.fr.verbalizers.verbalize_final import VerbalizeFinalFst
+        elif lang == 'se':
+            from nemo_text_processing.text_normalization.se.taggers.tokenize_and_classify import ClassifyFst
+            from nemo_text_processing.text_normalization.se.verbalizers.verbalize_final import VerbalizeFinalFst
         elif lang == 'sv':
             from nemo_text_processing.text_normalization.sv.taggers.tokenize_and_classify import ClassifyFst
             from nemo_text_processing.text_normalization.sv.verbalizers.verbalize_final import VerbalizeFinalFst
@@ -745,7 +748,7 @@ def parse_args():
     parser.add_argument(
         "--language",
         help="language",
-        choices=["en", "de", "es", "fr", "hu", "sv", "zh", "ar", "it", "hy", "ja", "hi", "ko", "vi", "pt"],
+        choices=["en", "de", "es", "fr", "hu", "se", "sv", "zh", "ar", "it", "hy", "ja", "hi", "ko", "vi", "pt"],
         default="en",
         type=str,
     )

nemo_text_processing/text_normalization/se/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/se/data/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/se/data/dates/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/se/data/dates/era_words.tsv

@@ -0,0 +1,4 @@
+o.Kr.	ovdal Kristtusa
+m.Kr.	maŋŋel Kristtusa
+oKr.	ovdal Kristtusa
+mKr.	maŋŋel Kristtusa

nemo_text_processing/text_normalization/se/data/dates/months_abbr.tsv

@@ -0,0 +1,13 @@
+ođđajagem	ođđajagemánnu
+ođđajagim	ođđajagimánnu
+guovvam	guovvamánnu
+njukčam	njukčamánnu
+cuoŋom	cuoŋománnu
+miessem	miessemánnu
+geassem	geassemánnu
+suoidnem	suoidnemánnu
+borgem	borgemánnu
+čakcam	čakčamánnu
+golggotm	golggotmánnu
+skábmam	skábmamánnu
+juovlam	juovlamánnu

nemo_text_processing/text_normalization/se/data/dates/months_gen.tsv

@@ -0,0 +1,12 @@
+ođđajagemánu	ođđajagemánnu
+guovvamánu	guovvamánnu
+njukčamánu	njukčamánnu
+cuoŋománu	cuoŋománnu
+miessemánu	miessemánnu
+geassemánu	geassemánnu
+suoidnemánu	suoidnemánnu
+borgemánu	borgemánnu
+čakčamánu	čakčamánnu
+golggotmánu	golggotmánnu
+skábmamánu	skábmamánnu
+juovlamánu	juovlamánnu

nemo_text_processing/text_normalization/se/data/dates/numbers.tsv

@@ -0,0 +1,12 @@
+1	ođđajagimánnu
+2	guovvamánnu
+3	njukčamánnu
+4	cuoŋománnu
+5	miessemánnu
+6	geassemánnu
+7	suoidnemánnu
+8	borgemánnu
+9	čakčamánnu
+10	golggotmánnu
+11	skábmamánnu
+12	juovlamánnu

nemo_text_processing/text_normalization/se/data/electronic/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/se/data/electronic/domain.tsv

@@ -0,0 +1,13 @@
+.com	čuokkis com
+.uk	čuokkis u k
+.fr	čuokkis f r
+.net	čuokkis net
+.br	čuokkis b r
+.in	čuokkis i n
+.ru	čuokkis r u
+.de	čuokkis d e
+.it	čuokkis i t
+.se	čuokkis s e
+.no	čuokkis n o
+.fi	čuokkis f i
+.es	čuokkis e s

nemo_text_processing/text_normalization/se/data/electronic/server_name.tsv

@@ -0,0 +1,18 @@
+gmail	g mail
+nvidia
+outlook
+hotmail
+yahoo
+live
+yandex
+orange
+wanadoo
+web
+comcast
+aol
+sametinget
+samediggi
+sámediggi
+ijahisidja	ijahis idja
+sajos
+salliparikka	salli parikka

nemo_text_processing/text_normalization/se/data/electronic/symbols.tsv

@@ -0,0 +1,21 @@
+.	čuokkis
+:	duppalčuokkis
+-	sáhcu
+!	čuorvvasmearka
+;	beallečuokkis
+'	apostrofa
+/	finjusáhcu
+?	gažaldatmearka
+,	rihkku
+_	deattuhusmearka
+#	fáddágilkor
+$	dollár
+%	proseanta
+&	ja
+*	asterisk
++	plussa
+=	ovttamađodatmearka
+^	sirkumfleaksa
+{	gurut bárroruohtu
+}	olgeš bárroruohtu
+~	tilde

nemo_text_processing/text_normalization/se/data/inflection/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/se/data/inflection/cardinal_alternatives.tsv

@@ -0,0 +1 @@
+6:i	guhtti

nemo_text_processing/text_normalization/se/data/inflection/case_suffixes.tsv

@@ -0,0 +1,10 @@
+nom_pl	t
+gen_sg	a
+gen_pl	id
+ill_sg	i
+ill_pl	ide
+loc_sg	s
+loc_pl	in
+com_sg	in
+com_pl	iguin
+ess	n

nemo_text_processing/text_normalization/se/data/inflection/jahki.tsv

@@ -0,0 +1,11 @@
+ess	jahkin
+com_pl	jagiiguin
+com_sg	jagiin
+acc_sg	jagi
+gen_sg	jage
+gen_pl	jagiid
+ill_pl	jagiide
+ill_sg	jahkái
+loc_sg	jagis
+nom_pl	jagit
+nom_sg	jagi

nemo_text_processing/text_normalization/se/data/inflection/ordinal_case_suffixes.tsv

@@ -0,0 +1,12 @@
+nom_pl	dat
+gen_sg	da
+acc_sg	da
+gen_pl	diid
+acc_pl	diid
+ill_sg	dii
+ill_pl	diidda
+loc_sg	dis
+loc_pl	diin
+com_sg	diin
+com_pl	diiguin
+ess	din

nemo_text_processing/text_normalization/se/data/inflection/simple.tsv

@@ -0,0 +1,9 @@
+:t	at
+:a	a
+:id	iid
+:i	ii
+:ida	iidda
+:s	is
+:in	iin
+:iguin	iiguin
+:n	in

nemo_text_processing/text_normalization/se/data/inflection/vahkku.tsv

@@ -0,0 +1,10 @@
+ess	vahkkun
+com_pl	vahkuiguin
+com_sg	vahkuin
+gen_sg	vahku
+gen_pl	vahkuid
+ill_pl	vahkuide
+ill_sg	vahkkui
+loc_sg	vahkus
+nom_pl	vahkut
+nom_sg	vahkku

nemo_text_processing/text_normalization/se/data/math_operations.tsv

@@ -0,0 +1,8 @@
++	plussa
+-	minus
+/	juohkit
+÷	juohkit
+:	juohkit
+×	geardde
+*	geardde
+·	geardde

nemo_text_processing/text_normalization/se/data/math_operations_no_equals.tsv

@@ -0,0 +1,4 @@
++	plussa
+-	sáhcu
+*	asterisk
+/	finjusáhcu

nemo_text_processing/text_normalization/se/data/math_prefix.tsv

@@ -0,0 +1,2 @@
++	plussa
+-	minus

nemo_text_processing/text_normalization/se/data/measure/__init__.py

@@ -0,0 +1,13 @@
+# Copyright (c) 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

nemo_text_processing/text_normalization/se/data/measure/greek_lower.tsv

@@ -0,0 +1,25 @@
+α	alfa
+β	beta
+γ	gamma
+δ	delta
+ε	epsilon
+ζ	zeta
+η	eta
+θ	theta
+ι	iota
+κ	kappa
+λ	lambda
+μ	mu
+ν	nu
+ξ	xi
+ο	omicron
+π	pi
+ρ	rho
+σ	sigma
+τ	tau
+υ	upsilon
+φ	phi
+χ	chi
+ψ	psi
+ω	omega
+

nemo_text_processing/text_normalization/se/data/measure/unit_genitive.tsv

@@ -0,0 +1,2 @@
+m	mehtera
+km	kilomehtera

nemo_text_processing/text_normalization/se/data/measure/unit_rate.tsv

@@ -0,0 +1,4 @@
+km/d	kilomehtera diimmus
+km/dii.	kilomehtera diimmus
+km/h	kilomehtera diimmus
+m/s	mehtera sekunddas