From 9995ba4852f9724f1098976b399c9b47a99eddc5 Mon Sep 17 00:00:00 2001 From: Jeanie Date: Mon, 2 Dec 2024 23:17:38 +0900 Subject: [PATCH 1/3] get this script working again and handle all the decoding error issues --- tools/i18n_helper.py | 131 ++++++++++++++++++++++++++----------------- 1 file changed, 81 insertions(+), 50 deletions(-) diff --git a/tools/i18n_helper.py b/tools/i18n_helper.py index 0970d5e96..37c4787c6 100755 --- a/tools/i18n_helper.py +++ b/tools/i18n_helper.py @@ -5,7 +5,6 @@ import sys import os import os.path -import pwd import re @@ -71,9 +70,9 @@ def __init__(self): self.args.xousdir = os.path.normpath(os.path.dirname(self.args.pdir)) self.args.program = os.path.basename(self.args.program) self.errfile = sys.stderr - # if end of string matches ' *EN*' (or some other lang) - self.temp_regex = re.compile("^.* \*[a-zA-Z]{2}\*$") - self.mt_regex = re.compile("^.* \*MT\*$") + # Regex to match strings ending with ' *EN*' or any other language code (e.g., ' *FR*') + self.temp_regex = re.compile(r"^.* \*[a-zA-Z]{2}\*$") + self.mt_regex = re.compile(r"^.* \*MT\*$") def out(self, *objects): print(*objects) @@ -96,15 +95,19 @@ def get_languages(self): return 1 # read keys in "rootkeys.backup_key": essential = "rootkeys.backup_key" - f = open( - root_keys, - ) - obj = json.load(f) - self.languages = [] - if essential in obj: - for key in obj[essential].keys(): - self.languages.append(key) - f.close() + try: + with open(root_keys, encoding='utf-8') as f: + obj = json.load(f) + self.languages = [] + if essential in obj: + for key in obj[essential].keys(): + self.languages.append(key) + except UnicodeDecodeError as e: + self.err(f"UnicodeDecodeError: {e}") + return 1 + except Exception as e: + self.err(f"Error: {e}") + return 1 return 0 def list_languages(self): @@ -150,11 +153,16 @@ def list_i18n_files(self): # other TAG def show_missing(self, i18n, is_manifest): i18n_path = os.path.join(self.args.xousdir, i18n) - f = open( - i18n_path, - ) - obj = json.load(f) - f.close() + try: + with open(i18n_path, encoding='utf-8') as f: + obj = json.load(f) + except UnicodeDecodeError as e: + self.err(f"UnicodeDecodeError: {e}") + return 1 + except Exception as e: + self.err(f"Error: {e}") + return 1 + jqpath = None translation = None for tag in obj.keys(): @@ -186,13 +194,20 @@ def show_missing(self, i18n, is_manifest): def stub_missing(self, i18n, is_manifest): i18n_path = os.path.join(self.args.xousdir, i18n) - f = open( - i18n_path, - ) - obj = json.load(f) - f.close() + try: + with open(i18n_path, encoding='utf-8') as f: + obj = json.load(f) + except UnicodeDecodeError as e: + self.err(f"UnicodeDecodeError: {e}") + return 1 + except Exception as e: + self.err(f"Error: {e}") + return 1 + jqpath = None translation = None + modified = False + for tag in obj.keys(): if is_manifest: appmenu = "appmenu." + tag @@ -205,21 +220,23 @@ def stub_missing(self, i18n, is_manifest): lang_path = jqpath + "." + lang if not lang in translation: if lang == "en-tts": - stub = translation['en']} + stub = translation['en'] else: stub = f"{translation['en']} *EN*" print(f"In {i18n}, at {lang_path} with en stub: {stub}") translation[lang] = stub - # blow 'em away. That's what git is for after all. - with open(i18n_path, "w") as f: - json.dump( - obj, - f, - ensure_ascii=False, - check_circular=False, - sort_keys=True, - indent=4, - ) + modified = True + if modified: + try: + with open(i18n_path, 'w', encoding='utf-8') as f: + json.dump(obj, f, ensure_ascii=False, check_circular=False, sort_keys=True, indent=4) + except UnicodeEncodeError as e: + self.err(f"UnicodeEncodeError: {e}") + return 1 + except Exception as e: + self.err(f"Error: {e}") + return 1 + return 0 def fill_missing_with_en_stub(self): @@ -255,12 +272,17 @@ def add_new_lang(self, i18n, is_manifest): if not os.path.exists(i18n_path_orig): os.rename(i18n_path, i18n_path_orig) self.verr('adding "%s" to %s' % (self.args.new_lang, i18n)) - orig_file = open( - i18n_path_orig, - ) - obj = json.load(orig_file) - orig_file.close() - new_file = open(i18n_path, "w") + + try: + with open(i18n_path_orig, encoding='utf-8') as orig_file: + obj = json.load(orig_file) + except UnicodeDecodeError as e: + self.err(f"UnicodeDecodeError: {e}") + return 1 + except Exception as e: + self.err(f"Error: {e}") + return 1 + translation = None for tag in obj.keys(): if is_manifest: @@ -273,15 +295,24 @@ def add_new_lang(self, i18n, is_manifest): if t_from != "🔇": t_new += self.from_hint translation[self.args.new_lang] = t_new - json.dump( - obj, - new_file, - ensure_ascii=False, - check_circular=False, - sort_keys=True, - indent=4, - ) - new_file.close() + + try: + with open(i18n_path, 'w', encoding='utf-8') as new_file: + json.dump( + obj, + new_file, + ensure_ascii=False, + check_circular=False, + sort_keys=True, + indent=4, + ) + except UnicodeEncodeError as e: + self.err(f"UnicodeEncodeError: {e}") + return 1 + except Exception as e: + self.err(f"Error: {e}") + return 1 + return 0 def new_lang(self): @@ -316,7 +347,7 @@ def run(self): rc = self.fill_missing_with_en_stub() elif self.args.new_lang: if not self.args.from_lang: - self.err("error: you must specify both --from-lang en --to-lang fr") + self.err("error: you must specify both --from-lang en --new-lang fr") rc = 1 else: self.get_languages() From 6f9e3717548ac62b170c619ffe67b3303230bcaf Mon Sep 17 00:00:00 2001 From: Jeanie Date: Mon, 2 Dec 2024 23:18:55 +0900 Subject: [PATCH 2/3] add back cheeky comment --- tools/i18n_helper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/i18n_helper.py b/tools/i18n_helper.py index 37c4787c6..a3debcf8e 100755 --- a/tools/i18n_helper.py +++ b/tools/i18n_helper.py @@ -227,6 +227,7 @@ def stub_missing(self, i18n, is_manifest): translation[lang] = stub modified = True if modified: + # blow 'em away. That's what git is for after all. try: with open(i18n_path, 'w', encoding='utf-8') as f: json.dump(obj, f, ensure_ascii=False, check_circular=False, sort_keys=True, indent=4) From 56e741207a343ee1d5672767c59a929c47be5e1b Mon Sep 17 00:00:00 2001 From: Jeanie Date: Mon, 2 Dec 2024 23:30:57 +0900 Subject: [PATCH 3/3] just handle errors, no need for the unicode specific stuff.. --- tools/i18n_helper.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/tools/i18n_helper.py b/tools/i18n_helper.py index a3debcf8e..8c57a2aa0 100755 --- a/tools/i18n_helper.py +++ b/tools/i18n_helper.py @@ -102,9 +102,6 @@ def get_languages(self): if essential in obj: for key in obj[essential].keys(): self.languages.append(key) - except UnicodeDecodeError as e: - self.err(f"UnicodeDecodeError: {e}") - return 1 except Exception as e: self.err(f"Error: {e}") return 1 @@ -156,9 +153,6 @@ def show_missing(self, i18n, is_manifest): try: with open(i18n_path, encoding='utf-8') as f: obj = json.load(f) - except UnicodeDecodeError as e: - self.err(f"UnicodeDecodeError: {e}") - return 1 except Exception as e: self.err(f"Error: {e}") return 1 @@ -197,9 +191,6 @@ def stub_missing(self, i18n, is_manifest): try: with open(i18n_path, encoding='utf-8') as f: obj = json.load(f) - except UnicodeDecodeError as e: - self.err(f"UnicodeDecodeError: {e}") - return 1 except Exception as e: self.err(f"Error: {e}") return 1 @@ -231,9 +222,6 @@ def stub_missing(self, i18n, is_manifest): try: with open(i18n_path, 'w', encoding='utf-8') as f: json.dump(obj, f, ensure_ascii=False, check_circular=False, sort_keys=True, indent=4) - except UnicodeEncodeError as e: - self.err(f"UnicodeEncodeError: {e}") - return 1 except Exception as e: self.err(f"Error: {e}") return 1 @@ -277,9 +265,6 @@ def add_new_lang(self, i18n, is_manifest): try: with open(i18n_path_orig, encoding='utf-8') as orig_file: obj = json.load(orig_file) - except UnicodeDecodeError as e: - self.err(f"UnicodeDecodeError: {e}") - return 1 except Exception as e: self.err(f"Error: {e}") return 1 @@ -307,9 +292,6 @@ def add_new_lang(self, i18n, is_manifest): sort_keys=True, indent=4, ) - except UnicodeEncodeError as e: - self.err(f"UnicodeEncodeError: {e}") - return 1 except Exception as e: self.err(f"Error: {e}") return 1