cfc/fix_encoding.py

import os
import re

# Windows-1252 mappings for bytes 0x80-0x9F
# These bytes appeared in the original UTF-8 stream but were interpreted as Windows-1252
WIN1252_MAP = {
    0x80: '\u20ac',  # Euro sign
    0x82: '\u201a',
    0x83: '\u0192',
    0x84: '\u201e',
    0x85: '\u2026',
    0x86: '\u2020',
    0x87: '\u2021',
    0x88: '\u02c6',
    0x89: '\u2030',
    0x8a: '\u0160',
    0x8b: '\u2039',
    0x8c: '\u0152',
    0x8e: '\u017d',
    0x91: '\u2018',
    0x92: '\u2019',
    0x93: '\u201c',
    0x94: '\u201d',
    0x95: '\u2022',
    0x96: '\u2013',
    0x97: '\u2014',
    0x98: '\u02dc',
    0x99: '\u2122',
    0x9a: '\u0161',
    0x9b: '\u203a',
    0x9c: '\u0153',
    0x9e: '\u017e',
    0x9f: '\u0178',
}

# Reverse map: Windows-1252 character -> original byte
WIN1252_REVERSE = {v: k for k, v in WIN1252_MAP.items()}


def garbled_to_bytes(text):
    """Convert garbled text back to original UTF-8 bytes.
    
    For each character in the garbled text:
    1. Try to encode as GBK -> gives us original UTF-8 bytes (usually 2 bytes)
    2. If it's a Windows-1252 character -> gives us the original single byte
    3. Try Latin-1 for bytes 0xA0-0xFF
    4. Otherwise, skip (unrecoverable)
    """
    result = bytearray()
    for c in text:
        # Try GBK encoding first (most common - 2 bytes)
        try:
            gbk_bytes = c.encode('gbk')
            result.extend(gbk_bytes)
            continue
        except (UnicodeEncodeError, LookupError):
            pass
        
        # Check if it's a Windows-1252 character (1 byte)
        if c in WIN1252_REVERSE:
            result.append(WIN1252_REVERSE[c])
            continue
        
        # Try Latin-1 for bytes 0xA0-0xFF
        try:
            latin1_bytes = c.encode('latin-1')
            if len(latin1_bytes) == 1 and latin1_bytes[0] >= 0x80:
                result.extend(latin1_bytes)
                continue
        except (UnicodeEncodeError, LookupError):
            pass
        
        # Unrecoverable character - use placeholder
        result.append(ord('?'))
    
    return bytes(result)


def fix_file(filepath):
    """Fix garbled Chinese in a single file."""
    with open(filepath, 'rb') as f:
        raw = f.read()
    
    # Detect and skip BOM
    bom = b''
    if raw.startswith(b'\xef\xbb\xbf'):
        bom = b'\xef\xbb\xbf'
        raw = raw[3:]
    
    # Decode as UTF-8
    text = raw.decode('utf-8')
    
    # Find all non-ASCII segments and try to fix them
    result_parts = []
    last_end = 0
    fixed_count = 0
    total_segments = 0
    
    for match in re.finditer(r'[^\x00-\x7F]+', text):
        garbled = match.group()
        start = match.start()
        end = match.end()
        total_segments += 1
        
        # Convert garbled text to original UTF-8 bytes
        original_bytes = garbled_to_bytes(garbled)
        
        # Try to decode as UTF-8
        try:
            fixed = original_bytes.decode('utf-8')
            result_parts.append(text[last_end:start])
            result_parts.append(fixed)
            last_end = end
            fixed_count += 1
        except UnicodeDecodeError:
            # If full decode fails, keep original
            pass
    
    result_parts.append(text[last_end:])
    new_text = ''.join(result_parts)
    
    if new_text != text:
        with open(filepath, 'wb') as f:
            f.write(bom + new_text.encode('utf-8'))
        return True, fixed_count, total_segments
    
    return False, 0, 0


def main():
    base_dir = r'E:\workspace\dan\zxyj\zxyj-backend\src\main\java\com\zxyj'
    
    total_files = 0
    fixed_files = 0
    total_fixed = 0
    total_segments = 0
    
    for root, dirs, files in os.walk(base_dir):
        for fname in sorted(files):
            if not fname.endswith('.java'):
                continue
            
            fpath = os.path.join(root, fname)
            total_files += 1
            
            try:
                changed, fc, ts = fix_file(fpath)
                if changed:
                    fixed_files += 1
                    total_fixed += fc
                    total_segments += ts
                    rel = os.path.relpath(fpath, base_dir)
                    print(f'Fixed: {rel} ({fc}/{ts} segments)')
            except Exception as e:
                print(f'Error: {os.path.relpath(fpath, base_dir)}: {e}')
    
    print(f'\nSummary: {fixed_files}/{total_files} files fixed, {total_fixed}/{total_segments} segments')


if __name__ == '__main__':
    main()