| // Copyright Mozilla Foundation. See the COPYRIGHT |
| // file at the top-level directory of this distribution. |
| // |
| // Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or |
| // https://www.apache.org/licenses/LICENSE-2.0> or the MIT license |
| // <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your |
| // option. This file may not be copied, modified, or distributed |
| // except according to those terms. |
| |
| use super::*; |
| use crate::ascii::*; |
| use crate::data::position; |
| use crate::handles::*; |
| use crate::variant::*; |
| |
| pub struct SingleByteDecoder { |
| table: &'static [u16; 128], |
| } |
| |
| impl SingleByteDecoder { |
| pub fn new(data: &'static [u16; 128]) -> VariantDecoder { |
| VariantDecoder::SingleByte(SingleByteDecoder { table: data }) |
| } |
| |
| pub fn max_utf16_buffer_length(&self, byte_length: usize) -> Option<usize> { |
| Some(byte_length) |
| } |
| |
| pub fn max_utf8_buffer_length_without_replacement(&self, byte_length: usize) -> Option<usize> { |
| byte_length.checked_mul(3) |
| } |
| |
| pub fn max_utf8_buffer_length(&self, byte_length: usize) -> Option<usize> { |
| byte_length.checked_mul(3) |
| } |
| |
| pub fn decode_to_utf8_raw( |
| &mut self, |
| src: &[u8], |
| dst: &mut [u8], |
| _last: bool, |
| ) -> (DecoderResult, usize, usize) { |
| let mut source = ByteSource::new(src); |
| let mut dest = Utf8Destination::new(dst); |
| 'outermost: loop { |
| match dest.copy_ascii_from_check_space_bmp(&mut source) { |
| CopyAsciiResult::Stop(ret) => return ret, |
| CopyAsciiResult::GoOn((mut non_ascii, mut handle)) => 'middle: loop { |
| // Start non-boilerplate |
| // |
| // Since the non-ASCIIness of `non_ascii` is hidden from |
| // the optimizer, it can't figure out that it's OK to |
| // statically omit the bound check when accessing |
| // `[u16; 128]` with an index |
| // `non_ascii as usize - 0x80usize`. |
| let mapped = |
| unsafe { *(self.table.get_unchecked(non_ascii as usize - 0x80usize)) }; |
| // let mapped = self.table[non_ascii as usize - 0x80usize]; |
| if mapped == 0u16 { |
| return ( |
| DecoderResult::Malformed(1, 0), |
| source.consumed(), |
| handle.written(), |
| ); |
| } |
| let dest_again = handle.write_bmp_excl_ascii(mapped); |
| // End non-boilerplate |
| match source.check_available() { |
| Space::Full(src_consumed) => { |
| return ( |
| DecoderResult::InputEmpty, |
| src_consumed, |
| dest_again.written(), |
| ); |
| } |
| Space::Available(source_handle) => { |
| match dest_again.check_space_bmp() { |
| Space::Full(dst_written) => { |
| return ( |
| DecoderResult::OutputFull, |
| source_handle.consumed(), |
| dst_written, |
| ); |
| } |
| Space::Available(mut destination_handle) => { |
| let (mut b, unread_handle) = source_handle.read(); |
| let source_again = unread_handle.commit(); |
| 'innermost: loop { |
| if b > 127 { |
| non_ascii = b; |
| handle = destination_handle; |
| continue 'middle; |
| } |
| // Testing on Haswell says that we should write the |
| // byte unconditionally instead of trying to unread it |
| // to make it part of the next SIMD stride. |
| let dest_again_again = destination_handle.write_ascii(b); |
| if b < 60 { |
| // We've got punctuation |
| match source_again.check_available() { |
| Space::Full(src_consumed_again) => { |
| return ( |
| DecoderResult::InputEmpty, |
| src_consumed_again, |
| dest_again_again.written(), |
| ); |
| } |
| Space::Available(source_handle_again) => { |
| match dest_again_again.check_space_bmp() { |
| Space::Full(dst_written_again) => { |
| return ( |
| DecoderResult::OutputFull, |
| source_handle_again.consumed(), |
| dst_written_again, |
| ); |
| } |
| Space::Available( |
| destination_handle_again, |
| ) => { |
| let (b_again, _unread_handle_again) = |
| source_handle_again.read(); |
| b = b_again; |
| destination_handle = |
| destination_handle_again; |
| continue 'innermost; |
| } |
| } |
| } |
| } |
| } |
| // We've got markup or ASCII text |
| continue 'outermost; |
| } |
| } |
| } |
| } |
| } |
| }, |
| } |
| } |
| } |
| |
| pub fn decode_to_utf16_raw( |
| &mut self, |
| src: &[u8], |
| dst: &mut [u16], |
| _last: bool, |
| ) -> (DecoderResult, usize, usize) { |
| let (pending, length) = if dst.len() < src.len() { |
| (DecoderResult::OutputFull, dst.len()) |
| } else { |
| (DecoderResult::InputEmpty, src.len()) |
| }; |
| let mut converted = 0usize; |
| 'outermost: loop { |
| match unsafe { |
| ascii_to_basic_latin( |
| src.as_ptr().add(converted), |
| dst.as_mut_ptr().add(converted), |
| length - converted, |
| ) |
| } { |
| None => { |
| return (pending, length, length); |
| } |
| Some((mut non_ascii, consumed)) => { |
| converted += consumed; |
| 'middle: loop { |
| // `converted` doesn't count the reading of `non_ascii` yet. |
| // Since the non-ASCIIness of `non_ascii` is hidden from |
| // the optimizer, it can't figure out that it's OK to |
| // statically omit the bound check when accessing |
| // `[u16; 128]` with an index |
| // `non_ascii as usize - 0x80usize`. |
| let mapped = |
| unsafe { *(self.table.get_unchecked(non_ascii as usize - 0x80usize)) }; |
| // let mapped = self.table[non_ascii as usize - 0x80usize]; |
| if mapped == 0u16 { |
| return ( |
| DecoderResult::Malformed(1, 0), |
| converted + 1, // +1 `for non_ascii` |
| converted, |
| ); |
| } |
| unsafe { |
| // The bound check has already been performed |
| *(dst.get_unchecked_mut(converted)) = mapped; |
| } |
| converted += 1; |
| // Next, handle ASCII punctuation and non-ASCII without |
| // going back to ASCII acceleration. Non-ASCII scripts |
| // use ASCII punctuation, so this avoid going to |
| // acceleration just for punctuation/space and then |
| // failing. This is a significant boost to non-ASCII |
| // scripts. |
| // TODO: Split out Latin converters without this part |
| // this stuff makes Latin script-conversion slower. |
| if converted == length { |
| return (pending, length, length); |
| } |
| let mut b = unsafe { *(src.get_unchecked(converted)) }; |
| 'innermost: loop { |
| if b > 127 { |
| non_ascii = b; |
| continue 'middle; |
| } |
| // Testing on Haswell says that we should write the |
| // byte unconditionally instead of trying to unread it |
| // to make it part of the next SIMD stride. |
| unsafe { |
| *(dst.get_unchecked_mut(converted)) = u16::from(b); |
| } |
| converted += 1; |
| if b < 60 { |
| // We've got punctuation |
| if converted == length { |
| return (pending, length, length); |
| } |
| b = unsafe { *(src.get_unchecked(converted)) }; |
| continue 'innermost; |
| } |
| // We've got markup or ASCII text |
| continue 'outermost; |
| } |
| } |
| } |
| } |
| } |
| } |
| |
| pub fn latin1_byte_compatible_up_to(&self, buffer: &[u8]) -> usize { |
| let mut bytes = buffer; |
| let mut total = 0; |
| loop { |
| if let Some((non_ascii, offset)) = validate_ascii(bytes) { |
| total += offset; |
| let mapped = unsafe { *(self.table.get_unchecked(non_ascii as usize - 0x80usize)) }; |
| if mapped != u16::from(non_ascii) { |
| return total; |
| } |
| total += 1; |
| bytes = &bytes[offset + 1..]; |
| } else { |
| return total; |
| } |
| } |
| } |
| } |
| |
| pub struct SingleByteEncoder { |
| table: &'static [u16; 128], |
| run_bmp_offset: usize, |
| run_byte_offset: usize, |
| run_length: usize, |
| } |
| |
| impl SingleByteEncoder { |
| pub fn new( |
| encoding: &'static Encoding, |
| data: &'static [u16; 128], |
| run_bmp_offset: u16, |
| run_byte_offset: u8, |
| run_length: u8, |
| ) -> Encoder { |
| Encoder::new( |
| encoding, |
| VariantEncoder::SingleByte(SingleByteEncoder { |
| table: data, |
| run_bmp_offset: run_bmp_offset as usize, |
| run_byte_offset: run_byte_offset as usize, |
| run_length: run_length as usize, |
| }), |
| ) |
| } |
| |
| pub fn max_buffer_length_from_utf16_without_replacement( |
| &self, |
| u16_length: usize, |
| ) -> Option<usize> { |
| Some(u16_length) |
| } |
| |
| pub fn max_buffer_length_from_utf8_without_replacement( |
| &self, |
| byte_length: usize, |
| ) -> Option<usize> { |
| Some(byte_length) |
| } |
| |
| #[inline(always)] |
| fn encode_u16(&self, code_unit: u16) -> Option<u8> { |
| // First, we see if the code unit falls into a run of consecutive |
| // code units that can be mapped by offset. This is very efficient |
| // for most non-Latin encodings as well as Latin1-ish encodings. |
| // |
| // For encodings that don't fit this pattern, the run (which may |
| // have the length of just one) just establishes the starting point |
| // for the next rule. |
| // |
| // Next, we do a forward linear search in the part of the index |
| // after the run. Even in non-Latin1-ish Latin encodings (except |
| // macintosh), the lower case letters are here. |
| // |
| // Next, we search the third quadrant up to the start of the run |
| // (upper case letters in Latin encodings except macintosh, in |
| // Greek and in KOI encodings) and then the second quadrant, |
| // except if the run stared before the third quadrant, we search |
| // the second quadrant up to the run. |
| // |
| // Last, we search the first quadrant, which has unused controls |
| // or punctuation in most encodings. This is bad for macintosh |
| // and IBM866, but those are rare. |
| |
| // Run of consecutive units |
| let unit_as_usize = code_unit as usize; |
| let offset = unit_as_usize.wrapping_sub(self.run_bmp_offset); |
| if offset < self.run_length { |
| return Some((128 + self.run_byte_offset + offset) as u8); |
| } |
| |
| // Search after the run |
| let tail_start = self.run_byte_offset + self.run_length; |
| if let Some(pos) = position(&self.table[tail_start..], code_unit) { |
| return Some((128 + tail_start + pos) as u8); |
| } |
| |
| if self.run_byte_offset >= 64 { |
| // Search third quadrant before the run |
| if let Some(pos) = position(&self.table[64..self.run_byte_offset], code_unit) { |
| return Some(((128 + 64) + pos) as u8); |
| } |
| |
| // Search second quadrant |
| if let Some(pos) = position(&self.table[32..64], code_unit) { |
| return Some(((128 + 32) + pos) as u8); |
| } |
| } else if let Some(pos) = position(&self.table[32..self.run_byte_offset], code_unit) { |
| // windows-1252, windows-874, ISO-8859-15 and ISO-8859-5 |
| // Search second quadrant before the run |
| return Some(((128 + 32) + pos) as u8); |
| } |
| |
| // Search first quadrant |
| if let Some(pos) = position(&self.table[..32], code_unit) { |
| return Some((128 + pos) as u8); |
| } |
| |
| None |
| } |
| |
| ascii_compatible_bmp_encoder_function!( |
| { |
| match self.encode_u16(bmp) { |
| Some(byte) => handle.write_one(byte), |
| None => { |
| return ( |
| EncoderResult::unmappable_from_bmp(bmp), |
| source.consumed(), |
| handle.written(), |
| ); |
| } |
| } |
| }, |
| bmp, |
| self, |
| source, |
| handle, |
| copy_ascii_to_check_space_one, |
| check_space_one, |
| encode_from_utf8_raw, |
| str, |
| Utf8Source, |
| true |
| ); |
| |
| pub fn encode_from_utf16_raw( |
| &mut self, |
| src: &[u16], |
| dst: &mut [u8], |
| _last: bool, |
| ) -> (EncoderResult, usize, usize) { |
| let (pending, length) = if dst.len() < src.len() { |
| (EncoderResult::OutputFull, dst.len()) |
| } else { |
| (EncoderResult::InputEmpty, src.len()) |
| }; |
| let mut converted = 0usize; |
| 'outermost: loop { |
| match unsafe { |
| basic_latin_to_ascii( |
| src.as_ptr().add(converted), |
| dst.as_mut_ptr().add(converted), |
| length - converted, |
| ) |
| } { |
| None => { |
| return (pending, length, length); |
| } |
| Some((mut non_ascii, consumed)) => { |
| converted += consumed; |
| 'middle: loop { |
| // `converted` doesn't count the reading of `non_ascii` yet. |
| match self.encode_u16(non_ascii) { |
| Some(byte) => { |
| unsafe { |
| *(dst.get_unchecked_mut(converted)) = byte; |
| } |
| converted += 1; |
| } |
| None => { |
| // At this point, we need to know if we |
| // have a surrogate. |
| let high_bits = non_ascii & 0xFC00u16; |
| if high_bits == 0xD800u16 { |
| // high surrogate |
| if converted + 1 == length { |
| // End of buffer. This surrogate is unpaired. |
| return ( |
| EncoderResult::Unmappable('\u{FFFD}'), |
| converted + 1, // +1 `for non_ascii` |
| converted, |
| ); |
| } |
| let second = |
| u32::from(unsafe { *src.get_unchecked(converted + 1) }); |
| if second & 0xFC00u32 != 0xDC00u32 { |
| return ( |
| EncoderResult::Unmappable('\u{FFFD}'), |
| converted + 1, // +1 `for non_ascii` |
| converted, |
| ); |
| } |
| // The next code unit is a low surrogate. |
| let astral: char = unsafe { |
| ::core::char::from_u32_unchecked( |
| (u32::from(non_ascii) << 10) + second |
| - (((0xD800u32 << 10) - 0x1_0000u32) + 0xDC00u32), |
| ) |
| }; |
| return ( |
| EncoderResult::Unmappable(astral), |
| converted + 2, // +2 `for non_ascii` and `second` |
| converted, |
| ); |
| } |
| if high_bits == 0xDC00u16 { |
| // Unpaired low surrogate |
| return ( |
| EncoderResult::Unmappable('\u{FFFD}'), |
| converted + 1, // +1 `for non_ascii` |
| converted, |
| ); |
| } |
| return ( |
| EncoderResult::unmappable_from_bmp(non_ascii), |
| converted + 1, // +1 `for non_ascii` |
| converted, |
| ); |
| } |
| } |
| // Next, handle ASCII punctuation and non-ASCII without |
| // going back to ASCII acceleration. Non-ASCII scripts |
| // use ASCII punctuation, so this avoid going to |
| // acceleration just for punctuation/space and then |
| // failing. This is a significant boost to non-ASCII |
| // scripts. |
| // TODO: Split out Latin converters without this part |
| // this stuff makes Latin script-conversion slower. |
| if converted == length { |
| return (pending, length, length); |
| } |
| let mut unit = unsafe { *(src.get_unchecked(converted)) }; |
| 'innermost: loop { |
| if unit > 127 { |
| non_ascii = unit; |
| continue 'middle; |
| } |
| // Testing on Haswell says that we should write the |
| // byte unconditionally instead of trying to unread it |
| // to make it part of the next SIMD stride. |
| unsafe { |
| *(dst.get_unchecked_mut(converted)) = unit as u8; |
| } |
| converted += 1; |
| if unit < 60 { |
| // We've got punctuation |
| if converted == length { |
| return (pending, length, length); |
| } |
| unit = unsafe { *(src.get_unchecked(converted)) }; |
| continue 'innermost; |
| } |
| // We've got markup or ASCII text |
| continue 'outermost; |
| } |
| } |
| } |
| } |
| } |
| } |
| } |
| |
| // Any copyright to the test code below this comment is dedicated to the |
| // Public Domain. http://creativecommons.org/publicdomain/zero/1.0/ |
| |
| #[cfg(test)] |
| mod tests { |
| use super::super::testing::*; |
| use super::super::*; |
| |
| #[test] |
| fn test_windows_1255_ca() { |
| decode(WINDOWS_1255, b"\xCA", "\u{05BA}"); |
| encode(WINDOWS_1255, "\u{05BA}", b"\xCA"); |
| } |
| |
| #[test] |
| fn test_ascii_punctuation() { |
| let bytes = b"\xC1\xF5\xF4\xFC \xE5\xDF\xED\xE1\xE9 \xDD\xED\xE1 \xF4\xE5\xF3\xF4. \xC1\xF5\xF4\xFC \xE5\xDF\xED\xE1\xE9 \xDD\xED\xE1 \xF4\xE5\xF3\xF4."; |
| let characters = "\u{0391}\u{03C5}\u{03C4}\u{03CC} \ |
| \u{03B5}\u{03AF}\u{03BD}\u{03B1}\u{03B9} \u{03AD}\u{03BD}\u{03B1} \ |
| \u{03C4}\u{03B5}\u{03C3}\u{03C4}. \u{0391}\u{03C5}\u{03C4}\u{03CC} \ |
| \u{03B5}\u{03AF}\u{03BD}\u{03B1}\u{03B9} \u{03AD}\u{03BD}\u{03B1} \ |
| \u{03C4}\u{03B5}\u{03C3}\u{03C4}."; |
| decode(WINDOWS_1253, bytes, characters); |
| encode(WINDOWS_1253, characters, bytes); |
| } |
| |
| #[test] |
| fn test_decode_malformed() { |
| decode( |
| WINDOWS_1253, |
| b"\xC1\xF5\xD2\xF4\xFC", |
| "\u{0391}\u{03C5}\u{FFFD}\u{03C4}\u{03CC}", |
| ); |
| } |
| |
| #[test] |
| fn test_encode_unmappables() { |
| encode( |
| WINDOWS_1253, |
| "\u{0391}\u{03C5}\u{2603}\u{03C4}\u{03CC}", |
| b"\xC1\xF5☃\xF4\xFC", |
| ); |
| encode( |
| WINDOWS_1253, |
| "\u{0391}\u{03C5}\u{1F4A9}\u{03C4}\u{03CC}", |
| b"\xC1\xF5💩\xF4\xFC", |
| ); |
| } |
| |
| #[test] |
| fn test_encode_unpaired_surrogates() { |
| encode_from_utf16( |
| WINDOWS_1253, |
| &[0x0391u16, 0x03C5u16, 0xDCA9u16, 0x03C4u16, 0x03CCu16], |
| b"\xC1\xF5�\xF4\xFC", |
| ); |
| encode_from_utf16( |
| WINDOWS_1253, |
| &[0x0391u16, 0x03C5u16, 0xD83Du16, 0x03C4u16, 0x03CCu16], |
| b"\xC1\xF5�\xF4\xFC", |
| ); |
| encode_from_utf16( |
| WINDOWS_1253, |
| &[0x0391u16, 0x03C5u16, 0x03C4u16, 0x03CCu16, 0xD83Du16], |
| b"\xC1\xF5\xF4\xFC�", |
| ); |
| } |
| |
| pub const HIGH_BYTES: &'static [u8; 128] = &[ |
| 0x80, 0x81, 0x82, 0x83, 0x84, 0x85, 0x86, 0x87, 0x88, 0x89, 0x8A, 0x8B, 0x8C, 0x8D, 0x8E, |
| 0x8F, 0x90, 0x91, 0x92, 0x93, 0x94, 0x95, 0x96, 0x97, 0x98, 0x99, 0x9A, 0x9B, 0x9C, 0x9D, |
| 0x9E, 0x9F, 0xA0, 0xA1, 0xA2, 0xA3, 0xA4, 0xA5, 0xA6, 0xA7, 0xA8, 0xA9, 0xAA, 0xAB, 0xAC, |
| 0xAD, 0xAE, 0xAF, 0xB0, 0xB1, 0xB2, 0xB3, 0xB4, 0xB5, 0xB6, 0xB7, 0xB8, 0xB9, 0xBA, 0xBB, |
| 0xBC, 0xBD, 0xBE, 0xBF, 0xC0, 0xC1, 0xC2, 0xC3, 0xC4, 0xC5, 0xC6, 0xC7, 0xC8, 0xC9, 0xCA, |
| 0xCB, 0xCC, 0xCD, 0xCE, 0xCF, 0xD0, 0xD1, 0xD2, 0xD3, 0xD4, 0xD5, 0xD6, 0xD7, 0xD8, 0xD9, |
| 0xDA, 0xDB, 0xDC, 0xDD, 0xDE, 0xDF, 0xE0, 0xE1, 0xE2, 0xE3, 0xE4, 0xE5, 0xE6, 0xE7, 0xE8, |
| 0xE9, 0xEA, 0xEB, 0xEC, 0xED, 0xEE, 0xEF, 0xF0, 0xF1, 0xF2, 0xF3, 0xF4, 0xF5, 0xF6, 0xF7, |
| 0xF8, 0xF9, 0xFA, 0xFB, 0xFC, 0xFD, 0xFE, 0xFF, |
| ]; |
| |
| fn decode_single_byte(encoding: &'static Encoding, data: &'static [u16; 128]) { |
| let mut with_replacement = [0u16; 128]; |
| let mut it = data.iter().enumerate(); |
| loop { |
| match it.next() { |
| Some((i, code_point)) => { |
| if *code_point == 0 { |
| with_replacement[i] = 0xFFFD; |
| } else { |
| with_replacement[i] = *code_point; |
| } |
| } |
| None => { |
| break; |
| } |
| } |
| } |
| |
| decode_to_utf16(encoding, HIGH_BYTES, &with_replacement[..]); |
| } |
| |
| fn encode_single_byte(encoding: &'static Encoding, data: &'static [u16; 128]) { |
| let mut with_zeros = [0u8; 128]; |
| let mut it = data.iter().enumerate(); |
| loop { |
| match it.next() { |
| Some((i, code_point)) => { |
| if *code_point == 0 { |
| with_zeros[i] = 0; |
| } else { |
| with_zeros[i] = HIGH_BYTES[i]; |
| } |
| } |
| None => { |
| break; |
| } |
| } |
| } |
| |
| encode_from_utf16(encoding, data, &with_zeros[..]); |
| } |
| |
| #[test] |
| fn test_single_byte_from_two_low_surrogates() { |
| let expectation = b"��"; |
| let mut output = [0u8; 40]; |
| let mut encoder = WINDOWS_1253.new_encoder(); |
| let (result, read, written, had_errors) = |
| encoder.encode_from_utf16(&[0xDC00u16, 0xDEDEu16], &mut output[..], true); |
| assert_eq!(result, CoderResult::InputEmpty); |
| assert_eq!(read, 2); |
| assert_eq!(written, expectation.len()); |
| assert!(had_errors); |
| assert_eq!(&output[..written], expectation); |
| } |
| |
| // These tests are so self-referential that they are pretty useless. |
| |
| // BEGIN GENERATED CODE. PLEASE DO NOT EDIT. |
| // Instead, please regenerate using generate-encoding-data.py |
| |
| #[test] |
| fn test_single_byte_decode() { |
| decode_single_byte(IBM866, &data::SINGLE_BYTE_DATA.ibm866); |
| decode_single_byte(ISO_8859_10, &data::SINGLE_BYTE_DATA.iso_8859_10); |
| if cfg!(miri) { |
| // Miri is too slow |
| return; |
| } |
| decode_single_byte(ISO_8859_13, &data::SINGLE_BYTE_DATA.iso_8859_13); |
| decode_single_byte(ISO_8859_14, &data::SINGLE_BYTE_DATA.iso_8859_14); |
| decode_single_byte(ISO_8859_15, &data::SINGLE_BYTE_DATA.iso_8859_15); |
| decode_single_byte(ISO_8859_16, &data::SINGLE_BYTE_DATA.iso_8859_16); |
| decode_single_byte(ISO_8859_2, &data::SINGLE_BYTE_DATA.iso_8859_2); |
| decode_single_byte(ISO_8859_3, &data::SINGLE_BYTE_DATA.iso_8859_3); |
| decode_single_byte(ISO_8859_4, &data::SINGLE_BYTE_DATA.iso_8859_4); |
| decode_single_byte(ISO_8859_5, &data::SINGLE_BYTE_DATA.iso_8859_5); |
| decode_single_byte(ISO_8859_6, &data::SINGLE_BYTE_DATA.iso_8859_6); |
| decode_single_byte(ISO_8859_7, &data::SINGLE_BYTE_DATA.iso_8859_7); |
| decode_single_byte(ISO_8859_8, &data::SINGLE_BYTE_DATA.iso_8859_8); |
| decode_single_byte(KOI8_R, &data::SINGLE_BYTE_DATA.koi8_r); |
| decode_single_byte(KOI8_U, &data::SINGLE_BYTE_DATA.koi8_u); |
| decode_single_byte(MACINTOSH, &data::SINGLE_BYTE_DATA.macintosh); |
| decode_single_byte(WINDOWS_1250, &data::SINGLE_BYTE_DATA.windows_1250); |
| decode_single_byte(WINDOWS_1251, &data::SINGLE_BYTE_DATA.windows_1251); |
| decode_single_byte(WINDOWS_1252, &data::SINGLE_BYTE_DATA.windows_1252); |
| decode_single_byte(WINDOWS_1253, &data::SINGLE_BYTE_DATA.windows_1253); |
| decode_single_byte(WINDOWS_1254, &data::SINGLE_BYTE_DATA.windows_1254); |
| decode_single_byte(WINDOWS_1255, &data::SINGLE_BYTE_DATA.windows_1255); |
| decode_single_byte(WINDOWS_1256, &data::SINGLE_BYTE_DATA.windows_1256); |
| decode_single_byte(WINDOWS_1257, &data::SINGLE_BYTE_DATA.windows_1257); |
| decode_single_byte(WINDOWS_1258, &data::SINGLE_BYTE_DATA.windows_1258); |
| decode_single_byte(WINDOWS_874, &data::SINGLE_BYTE_DATA.windows_874); |
| decode_single_byte(X_MAC_CYRILLIC, &data::SINGLE_BYTE_DATA.x_mac_cyrillic); |
| } |
| |
| #[test] |
| fn test_single_byte_encode() { |
| encode_single_byte(IBM866, &data::SINGLE_BYTE_DATA.ibm866); |
| encode_single_byte(ISO_8859_10, &data::SINGLE_BYTE_DATA.iso_8859_10); |
| if cfg!(miri) { |
| // Miri is too slow |
| return; |
| } |
| encode_single_byte(ISO_8859_13, &data::SINGLE_BYTE_DATA.iso_8859_13); |
| encode_single_byte(ISO_8859_14, &data::SINGLE_BYTE_DATA.iso_8859_14); |
| encode_single_byte(ISO_8859_15, &data::SINGLE_BYTE_DATA.iso_8859_15); |
| encode_single_byte(ISO_8859_16, &data::SINGLE_BYTE_DATA.iso_8859_16); |
| encode_single_byte(ISO_8859_2, &data::SINGLE_BYTE_DATA.iso_8859_2); |
| encode_single_byte(ISO_8859_3, &data::SINGLE_BYTE_DATA.iso_8859_3); |
| encode_single_byte(ISO_8859_4, &data::SINGLE_BYTE_DATA.iso_8859_4); |
| encode_single_byte(ISO_8859_5, &data::SINGLE_BYTE_DATA.iso_8859_5); |
| encode_single_byte(ISO_8859_6, &data::SINGLE_BYTE_DATA.iso_8859_6); |
| encode_single_byte(ISO_8859_7, &data::SINGLE_BYTE_DATA.iso_8859_7); |
| encode_single_byte(ISO_8859_8, &data::SINGLE_BYTE_DATA.iso_8859_8); |
| encode_single_byte(KOI8_R, &data::SINGLE_BYTE_DATA.koi8_r); |
| encode_single_byte(KOI8_U, &data::SINGLE_BYTE_DATA.koi8_u); |
| encode_single_byte(MACINTOSH, &data::SINGLE_BYTE_DATA.macintosh); |
| encode_single_byte(WINDOWS_1250, &data::SINGLE_BYTE_DATA.windows_1250); |
| encode_single_byte(WINDOWS_1251, &data::SINGLE_BYTE_DATA.windows_1251); |
| encode_single_byte(WINDOWS_1252, &data::SINGLE_BYTE_DATA.windows_1252); |
| encode_single_byte(WINDOWS_1253, &data::SINGLE_BYTE_DATA.windows_1253); |
| encode_single_byte(WINDOWS_1254, &data::SINGLE_BYTE_DATA.windows_1254); |
| encode_single_byte(WINDOWS_1255, &data::SINGLE_BYTE_DATA.windows_1255); |
| encode_single_byte(WINDOWS_1256, &data::SINGLE_BYTE_DATA.windows_1256); |
| encode_single_byte(WINDOWS_1257, &data::SINGLE_BYTE_DATA.windows_1257); |
| encode_single_byte(WINDOWS_1258, &data::SINGLE_BYTE_DATA.windows_1258); |
| encode_single_byte(WINDOWS_874, &data::SINGLE_BYTE_DATA.windows_874); |
| encode_single_byte(X_MAC_CYRILLIC, &data::SINGLE_BYTE_DATA.x_mac_cyrillic); |
| } |
| // END GENERATED CODE |
| } |
