/** * CJK Phrase Encoding * ------------------- * * Packs arbitrary binary data into a sequence of CJK Unified Ideographs. * * Each character encodes exactly 14 bits. The alphabet is the first 16,384 * code points starting at U+4E00 (一), ending at U+8DFF. This is a fixed * power-of-two subset chosen for maximum font coverage across desktop and * mobile operating systems. * * Power-of-two alignment (2^14 = 16,384) means there is no modulo bias, no * padding at the character level, and `codepoint = ALPHABET_START + index` * is a direct arithmetic mapping — no lookup table required. * * NOTE ON SPEC DEVIATION: The implementation spec cites ALPHABET_END as * U+87FF, which is inconsistent with ALPHABET_SIZE = 16384 (U+4E00 + 16384 * lands at U+8DFF, not U+87FF). The power-of-two invariant is load-bearing, * so this file treats U+8DFF as the correct upper bound. See the project * summary note for details. */ export const ALPHABET_START = 0x4e00; export const ALPHABET_SIZE = 16384; // 2^14 — every character packs exactly 14 bits export const ALPHABET_END = ALPHABET_START + ALPHABET_SIZE - 1; // 0x8DFF /** * Number of meaningful bits in the final CJK character of an encoding. * `totalBits = byteCount * 8`. If that is an exact multiple of 14 the last * character is fully used and this returns 0. Otherwise it returns the * remainder in the range 1..13. */ export function remainingBits(byteCount: number): number { const r = (byteCount * 8) % 14; return r; } /** * Encode a byte array into a CJK phrase string. * * Bytes are fed into a bit accumulator 8 bits at a time. Whenever the * accumulator holds at least 14 bits, the top 14 are extracted and emitted * as one character. Any leftover bits at the end become the most significant * bits of one final character, with the low bits zero-padded. The decoder * uses `remainingBits` to discard that padding. */ export function cjkEncode(bytes: Uint8Array): string { let bits = 0; // number of meaningful bits currently in the accumulator let buffer = 0; // bit accumulator — at peak can hold up to 21 bits const out: string[] = []; for (let i = 0; i < bytes.length; i++) { buffer = (buffer << 8) | bytes[i]; bits += 8; while (bits >= 14) { bits -= 14; const index = (buffer >>> bits) & 0x3fff; out.push(String.fromCodePoint(ALPHABET_START + index)); } } if (bits > 0) { // Leftover 1..13 bits: left-align into a 14-bit slot (zero-pad low bits) const index = (buffer << (14 - bits)) & 0x3fff; out.push(String.fromCodePoint(ALPHABET_START + index)); } return out.join(""); } /** * Decode a CJK phrase string back into bytes. `remainingBitsValue` must be * the value computed by `remainingBits(originalByteLength)` at encode time, * and is stored in the binary header at §2.1. * * Throws on any character outside the 16,384-entry alphabet, or if the * phrase length is inconsistent with `remainingBitsValue`. */ export function cjkDecode(phrase: string, remainingBitsValue: number): Uint8Array { let bits = 0; let buffer = 0; const out: number[] = []; // Spread handles surrogate pairs safely — every CJK ideograph in our // alphabet fits in a single BMP code unit but we spread defensively. const chars = [...phrase]; const lastIdx = chars.length - 1; for (let i = 0; i < chars.length; i++) { const cp = chars[i].codePointAt(0)!; const index = cp - ALPHABET_START; if (index < 0 || index >= ALPHABET_SIZE) { throw new Error( `INVALID_CHARACTER: U+${cp.toString(16).toUpperCase().padStart(4, "0")} at position ${i}` ); } const isLast = i === lastIdx; const bitsInThisChar = isLast && remainingBitsValue > 0 ? remainingBitsValue : 14; // The encoder left-aligned partial bits into the 14-bit slot; undo that // by right-shifting to discard the zero-padding on the last character. const value = isLast && remainingBitsValue > 0 ? index >>> (14 - remainingBitsValue) : index; buffer = (buffer << bitsInThisChar) | value; bits += bitsInThisChar; while (bits >= 8) { bits -= 8; out.push((buffer >>> bits) & 0xff); } } if (bits !== 0) { throw new Error("DECODE_ALIGNMENT_ERROR: residual bits after full decode"); } return new Uint8Array(out); } /** * True if every code point in `phrase` is within the 16,384-entry alphabet. * Used by the paste-validator before attempting a full decode. */ export function isValidPhraseCharset(phrase: string): boolean { for (const ch of phrase) { const cp = ch.codePointAt(0)!; if (cp < ALPHABET_START || cp > ALPHABET_END) return false; } return true; } /** * Decode the first `charCount` characters of a phrase assuming they all use * the full 14 bits (remaining_bits = 0), and return whatever bytes fully * accumulated. Residual sub-byte bits are discarded — this is the bootstrap * path used by the import pipeline to read the binary header before the * true `remaining_bits` value is available. * * Throws on any character outside the alphabet. Unlike `cjkDecode` this does * NOT enforce that the accumulator lands on an 8-bit boundary, because the * caller is slicing out of the middle of a longer phrase. */ export function cjkDecodeBootstrap(phrase: string, charCount: number): Uint8Array { let bits = 0; let buffer = 0; const out: number[] = []; const chars = [...phrase]; const n = Math.min(charCount, chars.length); for (let i = 0; i < n; i++) { const cp = chars[i].codePointAt(0)!; const index = cp - ALPHABET_START; if (index < 0 || index >= ALPHABET_SIZE) { throw new Error( `INVALID_CHARACTER: U+${cp.toString(16).toUpperCase().padStart(4, "0")} at position ${i}` ); } buffer = (buffer << 14) | index; bits += 14; while (bits >= 8) { bits -= 8; out.push((buffer >>> bits) & 0xff); } } return new Uint8Array(out); }