import os import re # Windows-1252 mappings for bytes 0x80-0x9F # These bytes appeared in the original UTF-8 stream but were interpreted as Windows-1252 WIN1252_MAP = { 0x80: '\u20ac', # Euro sign 0x82: '\u201a', 0x83: '\u0192', 0x84: '\u201e', 0x85: '\u2026', 0x86: '\u2020', 0x87: '\u2021', 0x88: '\u02c6', 0x89: '\u2030', 0x8a: '\u0160', 0x8b: '\u2039', 0x8c: '\u0152', 0x8e: '\u017d', 0x91: '\u2018', 0x92: '\u2019', 0x93: '\u201c', 0x94: '\u201d', 0x95: '\u2022', 0x96: '\u2013', 0x97: '\u2014', 0x98: '\u02dc', 0x99: '\u2122', 0x9a: '\u0161', 0x9b: '\u203a', 0x9c: '\u0153', 0x9e: '\u017e', 0x9f: '\u0178', } # Reverse map: Windows-1252 character -> original byte WIN1252_REVERSE = {v: k for k, v in WIN1252_MAP.items()} def garbled_to_bytes(text): """Convert garbled text back to original UTF-8 bytes. For each character in the garbled text: 1. Try to encode as GBK -> gives us original UTF-8 bytes (usually 2 bytes) 2. If it's a Windows-1252 character -> gives us the original single byte 3. Try Latin-1 for bytes 0xA0-0xFF 4. Otherwise, skip (unrecoverable) """ result = bytearray() for c in text: # Try GBK encoding first (most common - 2 bytes) try: gbk_bytes = c.encode('gbk') result.extend(gbk_bytes) continue except (UnicodeEncodeError, LookupError): pass # Check if it's a Windows-1252 character (1 byte) if c in WIN1252_REVERSE: result.append(WIN1252_REVERSE[c]) continue # Try Latin-1 for bytes 0xA0-0xFF try: latin1_bytes = c.encode('latin-1') if len(latin1_bytes) == 1 and latin1_bytes[0] >= 0x80: result.extend(latin1_bytes) continue except (UnicodeEncodeError, LookupError): pass # Unrecoverable character - use placeholder result.append(ord('?')) return bytes(result) def fix_file(filepath): """Fix garbled Chinese in a single file.""" with open(filepath, 'rb') as f: raw = f.read() # Detect and skip BOM bom = b'' if raw.startswith(b'\xef\xbb\xbf'): bom = b'\xef\xbb\xbf' raw = raw[3:] # Decode as UTF-8 text = raw.decode('utf-8') # Find all non-ASCII segments and try to fix them result_parts = [] last_end = 0 fixed_count = 0 total_segments = 0 for match in re.finditer(r'[^\x00-\x7F]+', text): garbled = match.group() start = match.start() end = match.end() total_segments += 1 # Convert garbled text to original UTF-8 bytes original_bytes = garbled_to_bytes(garbled) # Try to decode as UTF-8 try: fixed = original_bytes.decode('utf-8') result_parts.append(text[last_end:start]) result_parts.append(fixed) last_end = end fixed_count += 1 except UnicodeDecodeError: # If full decode fails, keep original pass result_parts.append(text[last_end:]) new_text = ''.join(result_parts) if new_text != text: with open(filepath, 'wb') as f: f.write(bom + new_text.encode('utf-8')) return True, fixed_count, total_segments return False, 0, 0 def main(): base_dir = r'E:\workspace\dan\zxyj\zxyj-backend\src\main\java\com\zxyj' total_files = 0 fixed_files = 0 total_fixed = 0 total_segments = 0 for root, dirs, files in os.walk(base_dir): for fname in sorted(files): if not fname.endswith('.java'): continue fpath = os.path.join(root, fname) total_files += 1 try: changed, fc, ts = fix_file(fpath) if changed: fixed_files += 1 total_fixed += fc total_segments += ts rel = os.path.relpath(fpath, base_dir) print(f'Fixed: {rel} ({fc}/{ts} segments)') except Exception as e: print(f'Error: {os.path.relpath(fpath, base_dir)}: {e}') print(f'\nSummary: {fixed_files}/{total_files} files fixed, {total_fixed}/{total_segments} segments') if __name__ == '__main__': main()