import 'dart:convert'; import 'dart:isolate'; import 'dart:typed_data'; import 'charset_tables.dart'; /// Text encodings the editor can read and write without loss. enum TextEncoding { utf8('UTF-8'), utf8Bom('UTF-8 BOM'), utf16le('UTF-16 LE'), utf16be('UTF-16 BE'), windows1250('Windows-1250'), windows1252('Windows-1252'), iso88592('ISO-8859-2'), latin1('ISO-8859-1'); final String label; const TextEncoding(this.label); /// The byte order mark written before the text, if any. List get bom => switch (this) { utf8Bom => const [0xef, 0xbb, 0xbf], utf16le => const [0xff, 0xfe], utf16be => const [0xfe, 0xff], _ => const [], }; } enum LineBreak { lf('LF', '\n'), crlf('CRLF', '\r\n'), cr('CR', '\r'); final String label, value; const LineBreak(this.label, this.value); } /// A character the target encoding cannot represent. class UnencodableText implements Exception { final TextEncoding encoding; final String sample; final int count; const UnencodableText(this.encoding, this.sample, this.count); @override String toString() => 'UnencodableText(${encoding.label}, $sample ×$count)'; } /// Line breaks used in a text. class LineBreakInfo { final int lf, crlf, cr; const LineBreakInfo(this.lf, this.crlf, this.cr); bool get mixed => [lf, crlf, cr].where((n) => n > 0).length > 1; LineBreak get dominant { if (crlf >= lf && crlf >= cr && crlf > 0) return LineBreak.crlf; if (cr > lf && cr > 0) return LineBreak.cr; return LineBreak.lf; } static LineBreakInfo of(String text) { var lf = 0, crlf = 0, cr = 0; for (var i = 0; i < text.length; i++) { final c = text.codeUnitAt(i); if (c == 0x0d) { if (i + 1 < text.length && text.codeUnitAt(i + 1) == 0x0a) { crlf++; i++; } else { cr++; } } else if (c == 0x0a) { lf++; } } return LineBreakInfo(lf, crlf, cr); } } /// The result of reading a file as text. class DecodedText { final String text; final TextEncoding encoding; final LineBreakInfo lineBreaks; final bool finalNewline; /// True when the bytes come back unchanged if the text is saved /// unchanged with [encoding] and the dominant line break. final bool exact; const DecodedText( this.text, this.encoding, this.lineBreaks, this.finalNewline, { required this.exact, }); } /// Whether [bytes] are most likely not text: NUL bytes or many control /// characters, unless the data is UTF-16. bool looksBinary(Uint8List bytes) { final n = bytes.length < 8192 ? bytes.length : 8192; if (n == 0) return false; if (_utf16Guess(bytes) != null) return false; var control = 0; for (var i = 0; i < n; i++) { final b = bytes[i]; if (b == 0) return true; if (b < 0x09 || (b > 0x0d && b < 0x20)) control++; } return control * 10 > n; } bool _startsWith(Uint8List b, List prefix) { if (b.length < prefix.length) return false; for (var i = 0; i < prefix.length; i++) { if (b[i] != prefix[i]) return false; } return true; } /// UTF-16 without a byte order mark, recognised by NUL bytes in every /// other position (mostly ASCII text). TextEncoding? _utf16Guess(Uint8List b) { final n = (b.length < 4096 ? b.length : 4096) & ~1; if (n < 4) return null; var evenZero = 0, oddZero = 0; for (var i = 0; i < n; i += 2) { if (b[i] == 0) evenZero++; if (b[i + 1] == 0) oddZero++; } final pairs = n ~/ 2; if (oddZero > pairs * 0.4 && evenZero < pairs * 0.05) { return TextEncoding.utf16le; } if (evenZero > pairs * 0.4 && oddZero < pairs * 0.05) { return TextEncoding.utf16be; } return null; } /// Detects the encoding: a byte order mark, then valid UTF-8, then UTF-16 /// without BOM. Returns null when the encoding cannot be known (legacy /// 8-bit text); the user then chooses, nothing is converted silently. TextEncoding? detectEncoding(Uint8List bytes) { if (_startsWith(bytes, TextEncoding.utf8Bom.bom)) return TextEncoding.utf8Bom; if (_startsWith(bytes, TextEncoding.utf16le.bom)) return TextEncoding.utf16le; if (_startsWith(bytes, TextEncoding.utf16be.bom)) return TextEncoding.utf16be; final utf16 = _utf16Guess(bytes); if (utf16 != null) return utf16; try { utf8.decode(bytes); return TextEncoding.utf8; } on FormatException { return null; } } String _decodeSingleByte(Uint8List b, List? high) { final units = Uint16List(b.length); for (var i = 0; i < b.length; i++) { final v = b[i]; if (v < 0x80 || high == null) { units[i] = v; } else { final c = high[v - 0x80]; if (c < 0) { throw FormatException('Byte 0x${v.toRadixString(16)} undefined', b, i); } units[i] = c; } } return String.fromCharCodes(units); } List? _table(TextEncoding e) => switch (e) { TextEncoding.windows1250 => windows1250High, TextEncoding.windows1252 => windows1252High, TextEncoding.iso88592 => iso88592High, _ => null, }; /// Decodes [bytes] as [encoding]. Throws [FormatException] for bytes that /// are not valid in that encoding. String decodeText(Uint8List bytes, TextEncoding encoding) { final bom = encoding.bom; final body = _startsWith(bytes, bom) ? Uint8List.sublistView(bytes, bom.length) : bytes; switch (encoding) { case TextEncoding.utf8: case TextEncoding.utf8Bom: return utf8.decode(body); case TextEncoding.utf16le: case TextEncoding.utf16be: if (body.length.isOdd) { throw const FormatException('UTF-16 data has an odd length'); } final le = encoding == TextEncoding.utf16le; final units = Uint16List(body.length ~/ 2); for (var i = 0; i < units.length; i++) { final a = body[2 * i], b = body[2 * i + 1]; units[i] = le ? a | (b << 8) : (a << 8) | b; } // Unpaired surrogates would not survive a round trip. for (var i = 0; i < units.length; i++) { final u = units[i]; if (u >= 0xd800 && u <= 0xdbff) { if (i + 1 >= units.length || units[i + 1] < 0xdc00 || units[i + 1] > 0xdfff) { throw FormatException('Unpaired surrogate', body, 2 * i); } i++; } else if (u >= 0xdc00 && u <= 0xdfff) { throw FormatException('Unpaired surrogate', body, 2 * i); } } return String.fromCharCodes(units); case TextEncoding.latin1: return _decodeSingleByte(body, null); case TextEncoding.windows1250: case TextEncoding.windows1252: case TextEncoding.iso88592: return _decodeSingleByte(body, _table(encoding)); } } /// Encodes [text] as [encoding], including its byte order mark. Throws /// [UnencodableText] when a character cannot be represented, instead of /// replacing it. Uint8List encodeText(String text, TextEncoding encoding) { final out = BytesBuilder(copy: false)..add(encoding.bom); switch (encoding) { case TextEncoding.utf8: case TextEncoding.utf8Bom: out.add(utf8.encode(text)); case TextEncoding.utf16le: case TextEncoding.utf16be: final le = encoding == TextEncoding.utf16le; final b = Uint8List(text.length * 2); for (var i = 0; i < text.length; i++) { final u = text.codeUnitAt(i); b[2 * i] = le ? u & 0xff : u >> 8; b[2 * i + 1] = le ? u >> 8 : u & 0xff; } out.add(b); case TextEncoding.latin1: case TextEncoding.windows1250: case TextEncoding.windows1252: case TextEncoding.iso88592: final table = _table(encoding); final reverse = { if (table != null) for (var i = 0; i < 128; i++) if (table[i] >= 0) table[i]: 0x80 + i, }; final b = Uint8List(text.length); String? sample; var bad = 0; for (var i = 0; i < text.length; i++) { final u = text.codeUnitAt(i); int? v; if (u < 0x80) { v = u; } else if (table == null) { v = u <= 0xff ? u : null; } else { v = reverse[u]; } if (v == null) { bad++; sample ??= String.fromCharCode(u); v = 0x3f; } b[i] = v; } if (bad > 0) throw UnencodableText(encoding, sample!, bad); out.add(b); } return out.takeBytes(); } bool _sameBytes(Uint8List a, Uint8List b) { if (a.length != b.length) return false; for (var i = 0; i < a.length; i++) { if (a[i] != b[i]) return false; } return true; } /// Decodes [bytes] for editing and records everything needed to write them /// back unchanged. Throws [FormatException] when [encoding] (or the /// detected one) does not fit, and [UnknownEncoding] when no encoding /// could be detected. DecodedText decodeDocument(Uint8List bytes, {TextEncoding? encoding}) { final enc = encoding ?? detectEncoding(bytes); if (enc == null) throw const UnknownEncoding(); final text = decodeText(bytes, enc); final breaks = LineBreakInfo.of(text); final finalNewline = text.endsWith('\n') || text.endsWith('\r'); bool exact; try { exact = !breaks.mixed && _sameBytes(encodeText(text, enc), bytes); } on UnencodableText { exact = false; } return DecodedText(text, enc, breaks, finalNewline, exact: exact); } class UnknownEncoding implements Exception { const UnknownEncoding(); } /// Joins editor lines (already split on any line break) with [lineBreak]. String withLineBreaks(String text, LineBreak lineBreak) { final normalized = text.replaceAll('\r\n', '\n').replaceAll('\r', '\n'); return lineBreak == LineBreak.lf ? normalized : normalized.replaceAll('\n', lineBreak.value); } /// [decodeDocument] on a background isolate for large files. Top-level so /// that the closure sent to the isolate captures only its arguments. Future decodeInBackground( Uint8List bytes, { TextEncoding? encoding, }) => bytes.length > 256 * 1024 ? Isolate.run(() => decodeDocument(bytes, encoding: encoding)) : Future.value(decodeDocument(bytes, encoding: encoding)); /// [encodeText] on a background isolate for large texts. Future encodeInBackground(String text, TextEncoding encoding) => text.length > 256 * 1024 ? Isolate.run(() => encodeText(text, encoding)) : Future.value(encodeText(text, encoding));