close
Skip to content

update run_evaluate script for cased itn #164

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tbartley94 merged 2 commits into from
Apr 25, 2024
Merged

update run_evaluate script for cased itn #164

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8450f63
update run_evaluate script for cased itn
mgrafu Apr 24, 2024
3406db8
[pre-commit.ci] auto fixes from pre-commit.com hooks
pre-commit-ci[bot] Apr 24, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 14 additions & 3 deletions nemo_text_processing/inverse_text_normalization/run_evaluate.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,14 @@ def parse_args():
parser = ArgumentParser()
parser.add_argument("--input", help="input file path", type=str)
parser.add_argument(
"--lang", help="language", choices=['en', 'de', 'es', 'pt', 'ru', 'fr', 'vi', 'hy'], default="en", type=str
"--lang",
help="language",
choices=["ar", "de", "en", "es", "es_en", "fr", "hy", "mr", "pt", "ru", "sv", "vi", "zh"],
default="en",
type=str,
)
parser.add_argument("--input_case", choices=["lower_cased", "cased"])
parser.add_argument("--output_case", choices=["lower_cased", "cased"])
parser.add_argument(
"--cat",
dest="category",
Expand All @@ -54,10 +60,15 @@ def parse_args():
if args.lang == 'en':
from nemo_text_processing.inverse_text_normalization.en.clean_eval_data import filter_loaded_data
file_path = args.input
inverse_normalizer = InverseNormalizer(lang=args.lang)
inverse_normalizer = InverseNormalizer(lang=args.lang, input_case=args.input_case)

print("Loading training data: " + file_path)
training_data = load_files([file_path])
if args.output_case == "lower_cased":
to_lower = True
elif args.output_case == "cased":
to_lower = False

training_data = load_files([file_path], to_lower=to_lower)

if args.filter:
training_data = filter_loaded_data(training_data)
Expand Down
11 changes: 6 additions & 5 deletions nemo_text_processing/text_normalization/data_loader_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,7 @@
]


def _load_kaggle_text_norm_file(file_path: str) -> List[Instance]:
def _load_kaggle_text_norm_file(file_path: str, to_lower: bool) -> List[Instance]:
"""
https://www.kaggle.com/richardwilliamsproat/text-normalization-for-english-russian-and-polish
Loads text file in the Kaggle Google text normalization file format: <semiotic class>\t<unnormalized text>\t<`self` if trivial class or normalized text>
Expand Down Expand Up @@ -76,8 +76,9 @@ def _load_kaggle_text_norm_file(file_path: str) -> List[Instance]:
res.append(Instance(token_type=EOS_TYPE, un_normalized="", normalized=""))
else:
l_type, l_token, l_normalized = parts
l_token = l_token.lower()
l_normalized = l_normalized.lower()
if to_lower:
l_token = l_token.lower()
l_normalized = l_normalized.lower()

if l_type == PLAIN_TYPE:
res.append(Instance(token_type=l_type, un_normalized=l_token, normalized=l_token))
Expand All @@ -86,7 +87,7 @@ def _load_kaggle_text_norm_file(file_path: str) -> List[Instance]:
return res


def load_files(file_paths: List[str], load_func=_load_kaggle_text_norm_file) -> List[Instance]:
def load_files(file_paths: List[str], load_func=_load_kaggle_text_norm_file, to_lower: bool = True) -> List[Instance]:
"""
Load given list of text files using the `load_func` function.

Expand All @@ -98,7 +99,7 @@ def load_files(file_paths: List[str], load_func=_load_kaggle_text_norm_file) ->
"""
res = []
for file_path in file_paths:
res.extend(load_func(file_path=file_path))
res.extend(load_func(file_path=file_path, to_lower=to_lower))
return res


Expand Down