|
|
@@ -22,8 +22,8 @@ extern crate test;
|
|
|
// Note: the method described in the original article (using Huffman trees and thus only probabilities that are negative powers of 2) would eliminate the need for inefficient multiplications of arbitrary bignums, which turned out (after developing this prototype) to be too inefficient. This is probably the biggest single thing / tradeoff we can do to attain acceptable performance. Any other ideas?
|
|
|
|
|
|
// Next steps / ideas:
|
|
|
-// - Optimize until it actually becomes usable (see above)
|
|
|
-// - Develop a message / packet format (encrypted, authenticated, numbered, delimited).
|
|
|
+// - Optimize until it actually becomes usable (see above - specialize everything to powers of 2, then start profiling in earnest)
|
|
|
+// - Obtain independent review of the crypto, the message format and everything else
|
|
|
// - Mark some nonterminals as non-coding to make messages somewhat robust against tampering?
|
|
|
|
|
|
#[macro_use]
|
|
|
@@ -63,6 +63,14 @@ use std::io::BufReader;
|
|
|
// use c2_chacha::stream_cipher::{NewStreamCipher, SyncStreamCipher, SyncStreamCipherSeek};
|
|
|
// use c2_chacha::{ChaCha12, ChaCha20};
|
|
|
|
|
|
+use aes::block_cipher::generic_array::GenericArray;
|
|
|
+use aes::block_cipher::{BlockCipher, NewBlockCipher};
|
|
|
+use aes::Aes128;
|
|
|
+use block_modes::{BlockMode, Cbc};
|
|
|
+use block_modes::block_padding::Pkcs7;
|
|
|
+type Aes128Cbc = Cbc<Aes128, Pkcs7>;
|
|
|
+use sha3::{Digest, Sha3_256};
|
|
|
+
|
|
|
use std::cmp::min;
|
|
|
|
|
|
// Not regex::bytes::Regex: used for reading YAML which is always unicode
|
|
|
@@ -814,7 +822,8 @@ impl<'a> PrefixfreeDecoder<'a> {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
-
|
|
|
+/// The plain decoder, without the requirement for the grammar to be prefix-free.
|
|
|
+/// Instead, it relies on the sentences being delimited externally.
|
|
|
impl<'a> Decoder<'a> {
|
|
|
pub fn new(scfg: &'a SCFG, inputs: &'a mut dyn Iterator<Item = Vec<u8>>) -> Decoder<'a> {
|
|
|
|
|
|
@@ -828,6 +837,13 @@ impl<'a> Decoder<'a> {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+ /// @precondition: use only when remaining_input is empty
|
|
|
+ pub fn refill(self: &'a mut Self, additional_inputs: &'a mut dyn Iterator<Item = Vec<u8>>) {
|
|
|
+ let head = additional_inputs.next().unwrap();
|
|
|
+ self.input = head;
|
|
|
+ self.remaining_inputs = Box::new(additional_inputs);
|
|
|
+ }
|
|
|
+
|
|
|
pub fn decode(self: &mut Self) -> Result<BigRational, anyhow::Error> {
|
|
|
loop {
|
|
|
let k = step_decode(&self.grammar, &mut self.state, &self.input);
|
|
|
@@ -839,6 +855,9 @@ impl<'a> Decoder<'a> {
|
|
|
Ok(DecodeStepResult::Finished {
|
|
|
choices
|
|
|
}) => {
|
|
|
+ // in case we want to refill with additional inputs
|
|
|
+ // TODO: test whether this works as intended; apply
|
|
|
+ self.state = vec![DecoderStateEntry::from_previous_choices(choices.clone())];
|
|
|
match self.remaining_inputs.next() {
|
|
|
None => {
|
|
|
let l = compute_bounds(&self.grammar, &choices).l;
|
|
|
@@ -847,14 +866,14 @@ impl<'a> Decoder<'a> {
|
|
|
Some(input) => {
|
|
|
let next = input;
|
|
|
self.input = next;
|
|
|
- self.state = vec![DecoderStateEntry::from_previous_choices(choices)];
|
|
|
}
|
|
|
}
|
|
|
}
|
|
|
|
|
|
- Ok(DecodeStepResult::PrefixParsed {
|
|
|
- ..
|
|
|
- }) => {
|
|
|
+ // Since the grammar is unambiguous, and this is the parser that must
|
|
|
+ // parse the entire input (language not assumed to be prefix-free),
|
|
|
+ // it must be fatal.
|
|
|
+ Ok(DecodeStepResult::PrefixParsed { .. }) => {
|
|
|
return Err(anyhow::Error::msg("Only a prefix was parsed"));
|
|
|
}
|
|
|
|
|
|
@@ -985,6 +1004,244 @@ impl Iterator for DeterministicPseudoRandoms {
|
|
|
}
|
|
|
}
|
|
|
|
|
|
+
|
|
|
+
|
|
|
+/***********************
|
|
|
+ * MESSAGE <-> PACKETS *
|
|
|
+ ***********************/
|
|
|
+
|
|
|
+pub struct Packetizer<'a> {
|
|
|
+ rng: rand_chacha::ChaCha8Rng,
|
|
|
+ aes_key: GenericArray<u8, <aes::Aes128 as aes::NewBlockCipher>::KeySize>,
|
|
|
+ messages_iter: Box<&'a mut dyn Iterator<Item = Vec<u8>>>,
|
|
|
+ conversation_id: &'a [u8; 8],
|
|
|
+ /// only 6 bytes are used
|
|
|
+ sequence_no: usize,
|
|
|
+}
|
|
|
+
|
|
|
+fn xor_blocks(block1: &[u8], block2: &[u8]) -> Vec<u8>
|
|
|
+{
|
|
|
+ let mut out : Vec<u8> = Vec::with_capacity(block1.len());
|
|
|
+ block1.iter().zip(block2.iter()).for_each(
|
|
|
+ |(x1, x2)|
|
|
|
+ {
|
|
|
+ out.push(x1.clone() ^ x2.clone());
|
|
|
+ }
|
|
|
+ );
|
|
|
+ return out
|
|
|
+}
|
|
|
+
|
|
|
+/// HMAC, general definition: https://tools.ietf.org/html/rfc2104
|
|
|
+/// "To compute HMAC over the data `text' we perform
|
|
|
+/// H(K XOR opad, H(K XOR ipad, text))"
|
|
|
+/// Truncating HMACs is permissible.
|
|
|
+fn hmac_sha3(key: &[u8], msg: &[u8]) -> Vec<u8> {
|
|
|
+ let ipad = vec![0x36; 16];
|
|
|
+ let opad = vec![0x5C; 16];
|
|
|
+ let mut hash_inner = Sha3_256::new();
|
|
|
+ hash_inner.update(xor_blocks(key, &ipad[..]));
|
|
|
+ hash_inner.update(msg.clone());
|
|
|
+ let result_inner = hash_inner.finalize();
|
|
|
+ let mut hash_outer = Sha3_256::new();
|
|
|
+ hash_outer.update(xor_blocks(key, &opad[..]));
|
|
|
+ hash_outer.update(result_inner);
|
|
|
+ let result_outer : Vec<u8> = hash_outer.finalize().to_vec();
|
|
|
+ result_outer
|
|
|
+}
|
|
|
+
|
|
|
+#[derive(Clone, Debug)]
|
|
|
+struct Header {
|
|
|
+ msg_size: u16,
|
|
|
+ conversation_id: Vec<u8>,
|
|
|
+ sequence_no: usize,
|
|
|
+ enc_hdr_as_vec: Vec<u8>,
|
|
|
+}
|
|
|
+
|
|
|
+#[derive(Clone, Debug)]
|
|
|
+struct Packet {
|
|
|
+ header: Header,
|
|
|
+ payload: Vec<u8>,
|
|
|
+}
|
|
|
+
|
|
|
+#[derive(Clone, Debug)]
|
|
|
+enum DepacketizeError {
|
|
|
+ HeaderHmacMismatch,
|
|
|
+ HeaderAndMessageHmacMismatch,
|
|
|
+ TooShortForHeader,
|
|
|
+ TooShortForHeaderAndMessageHmac,
|
|
|
+ TooShortForMessage,
|
|
|
+}
|
|
|
+
|
|
|
+/// See if it is already possible to glean a packet header from the given input,
|
|
|
+/// which may just be a prefix of the full input.
|
|
|
+/// Intended to be used to detect the start of a packet, to "synchronize" decoder
|
|
|
+/// with message stream.
|
|
|
+fn try_read_packet_header(key: &[u8], input: &[u8]) -> Result<Header, DepacketizeError> {
|
|
|
+
|
|
|
+ if input.len() < 40 {
|
|
|
+ return Err(DepacketizeError::TooShortForHeader);
|
|
|
+ }
|
|
|
+
|
|
|
+ let (encrypted_nonce, input) = input.split_at(16);
|
|
|
+ let (xor_hdr, input) = input.split_at(16);
|
|
|
+ let (hmac_hdr, _) = input.split_at(8);
|
|
|
+
|
|
|
+ let my_hmac_hdr : Vec<u8> = hmac_sha3(&key[..], &xor_hdr[..]);
|
|
|
+ if hmac_hdr != &my_hmac_hdr[..8] {
|
|
|
+ return Err(DepacketizeError::HeaderHmacMismatch);
|
|
|
+ }
|
|
|
+
|
|
|
+ let cipher = Aes128::new(GenericArray::from_slice(key));
|
|
|
+ let mut unencrypted_nonce = GenericArray::clone_from_slice(&encrypted_nonce);
|
|
|
+ cipher.decrypt_block(&mut unencrypted_nonce);
|
|
|
+ let hdr = xor_blocks(&unencrypted_nonce, xor_hdr);
|
|
|
+
|
|
|
+ let hdrx = &hdr[..];
|
|
|
+ let (conv_id, hdrx) = hdrx.split_at(8);
|
|
|
+ let (size, seq_no) = hdrx.split_at(2);
|
|
|
+
|
|
|
+ let mut seq_no8 : [u8; 8] = Default::default();
|
|
|
+ seq_no8[2..].copy_from_slice(&seq_no);
|
|
|
+ let seq_no = usize::from_be_bytes(seq_no8);
|
|
|
+
|
|
|
+ let mut size2: [u8; 2] = Default::default();
|
|
|
+ size2.copy_from_slice(&size[..]);
|
|
|
+ let size = u16::from_be_bytes(size2);
|
|
|
+
|
|
|
+ let mut enc_hdr_as_vec = vec![];
|
|
|
+ enc_hdr_as_vec.extend_from_slice(&encrypted_nonce[..]);
|
|
|
+ enc_hdr_as_vec.extend_from_slice(&xor_hdr[..]);
|
|
|
+
|
|
|
+ Ok(Header{
|
|
|
+ msg_size: size,
|
|
|
+ conversation_id: conv_id.to_vec(),
|
|
|
+ sequence_no: seq_no,
|
|
|
+ enc_hdr_as_vec: enc_hdr_as_vec.to_vec(),
|
|
|
+ })
|
|
|
+}
|
|
|
+
|
|
|
+fn try_depacketize(key: &[u8], input: &[u8]) -> Result<Packet, DepacketizeError> {
|
|
|
+
|
|
|
+ match try_read_packet_header(key, input) {
|
|
|
+ Ok(header) => {
|
|
|
+ let size = header.msg_size;
|
|
|
+
|
|
|
+ if input.len() < 48 {
|
|
|
+ return Err(DepacketizeError::TooShortForHeaderAndMessageHmac);
|
|
|
+ }
|
|
|
+
|
|
|
+ let (_, input) = input.split_at(40);
|
|
|
+ let (hmac_hdr_msg, input) = input.split_at(8);
|
|
|
+
|
|
|
+ if input.len() < size.into() {
|
|
|
+ return Err(DepacketizeError::TooShortForMessage);
|
|
|
+ }
|
|
|
+
|
|
|
+ let (iv, ciphertext) = input.split_at(16);
|
|
|
+
|
|
|
+ let mut enc_hdr_msg = vec![];
|
|
|
+ enc_hdr_msg.extend_from_slice(&header.enc_hdr_as_vec[..]);
|
|
|
+ enc_hdr_msg.extend_from_slice(&iv[..]);
|
|
|
+ enc_hdr_msg.extend_from_slice(&ciphertext[..size as usize - iv.len()]);
|
|
|
+ let my_hmac_hdr_msg : Vec<u8> = hmac_sha3(&key[..], &enc_hdr_msg[..]);
|
|
|
+
|
|
|
+ if hmac_hdr_msg != &my_hmac_hdr_msg[..8] {
|
|
|
+ return Err(DepacketizeError::HeaderAndMessageHmacMismatch);
|
|
|
+ }
|
|
|
+
|
|
|
+ let cipher = Aes128Cbc::new_var(&key, &iv).unwrap();
|
|
|
+ let decrypted_ciphertext = cipher.decrypt_vec(&ciphertext).unwrap();
|
|
|
+ Ok(Packet{ payload: decrypted_ciphertext, header })
|
|
|
+ }
|
|
|
+ Err(e) => {
|
|
|
+ Err(e)
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+/// Message format:
|
|
|
+/// - header:
|
|
|
+/// - AES128block(key, nonce) (16 bytes)
|
|
|
+/// - header, encrypted by xor'ing with nonce
|
|
|
+/// - conversation id (8 bytes)
|
|
|
+/// - ciphertext size
|
|
|
+/// - HMAC (encrypted header) truncated to 8 bytes
|
|
|
+/// - HMAC (encrypted nonce || encrypted header || iv || ciphertext), truncated to 8 bytes
|
|
|
+/// - payload:
|
|
|
+/// - random iv (16 bytes)
|
|
|
+/// - ciphertext, consisting of message encrypted as
|
|
|
+/// AES128Cbc(key, iv=encrypted nonce, message)
|
|
|
+impl<'a> Iterator for Packetizer<'a> {
|
|
|
+ type Item = Vec<u8>;
|
|
|
+
|
|
|
+ fn next(&mut self) -> Option<Vec<u8>> {
|
|
|
+
|
|
|
+ match self.messages_iter.next() {
|
|
|
+ Some(message) => {
|
|
|
+ let key = self.aes_key.clone();
|
|
|
+
|
|
|
+ let mut nonce_vec = vec![0u8; 16];
|
|
|
+ self.rng.fill(&mut nonce_vec[..]);
|
|
|
+ let mut nonce_array = GenericArray::clone_from_slice(&nonce_vec[..]);
|
|
|
+
|
|
|
+ let cipher_n = Aes128::new(&key);
|
|
|
+ let unencrypted_nonce = nonce_array.clone();
|
|
|
+ cipher_n.encrypt_block(&mut nonce_array);
|
|
|
+ let mut encrypted_nonce = nonce_array.to_vec();
|
|
|
+
|
|
|
+ let mut iv = vec![0u8; 16];
|
|
|
+ self.rng.fill(&mut iv[..]);
|
|
|
+ let cipher = Aes128Cbc::new_var(&key, &iv).unwrap();
|
|
|
+ let mut ciphertext = cipher.encrypt_vec(&message[..]);
|
|
|
+
|
|
|
+ let mut hdr = vec![];
|
|
|
+ // 2 bytes are plenty, considering the inefficiency of the encoding.
|
|
|
+ let size : u16 = ciphertext.len() as u16 + iv.len() as u16;
|
|
|
+ let mut size_be = u16::to_be_bytes(size).to_vec();
|
|
|
+
|
|
|
+ // 8 bytes for conversation id
|
|
|
+ hdr.extend_from_slice(self.conversation_id);
|
|
|
+ // 2 bytes for size of message
|
|
|
+ hdr.append(&mut size_be);
|
|
|
+ // 6 bytes for sequence number
|
|
|
+ hdr.extend_from_slice(&usize::to_be_bytes(self.sequence_no)[2..8]);
|
|
|
+
|
|
|
+ self.sequence_no += 1;
|
|
|
+
|
|
|
+ let mut xor_hdr = vec![];
|
|
|
+ for i in 0 .. hdr.len() {
|
|
|
+ xor_hdr.push(hdr[i] ^ unencrypted_nonce[i]);
|
|
|
+ }
|
|
|
+
|
|
|
+ let mut hmac_hdr : Vec<u8> = hmac_sha3(&key[..], &xor_hdr[..]);
|
|
|
+ let mut enc_hdr_msg = vec![];
|
|
|
+ enc_hdr_msg.extend_from_slice(&encrypted_nonce[..]);
|
|
|
+ enc_hdr_msg.extend_from_slice(&xor_hdr[..]);
|
|
|
+ enc_hdr_msg.extend_from_slice(&iv[..]);
|
|
|
+ enc_hdr_msg.extend_from_slice(&ciphertext[..]);
|
|
|
+ let mut hmac_hdr_msg : Vec<u8> = hmac_sha3(&key[..], &enc_hdr_msg[..]);
|
|
|
+
|
|
|
+ hmac_hdr.truncate(8);
|
|
|
+ hmac_hdr_msg.truncate(8);
|
|
|
+
|
|
|
+ let mut enc_msg = vec![];
|
|
|
+ enc_msg.append(&mut encrypted_nonce);
|
|
|
+ enc_msg.append(&mut xor_hdr.clone());
|
|
|
+ enc_msg.append(&mut hmac_hdr.clone());
|
|
|
+ enc_msg.append(&mut hmac_hdr_msg.clone());
|
|
|
+ enc_msg.append(&mut iv);
|
|
|
+ enc_msg.append(&mut ciphertext);
|
|
|
+
|
|
|
+ Some(enc_msg)
|
|
|
+ }
|
|
|
+ None => {
|
|
|
+ None
|
|
|
+ }
|
|
|
+ }
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+
|
|
|
trait GetOk<'a, I: std::fmt::Display, R> {
|
|
|
fn get_ok(&self, index: I) -> std::result::Result<&R, anyhow::Error>;
|
|
|
fn has(&self, index: I) -> bool;
|
|
|
@@ -1547,6 +1804,37 @@ mod tests {
|
|
|
}
|
|
|
|
|
|
#[test]
|
|
|
+ fn packetizer() {
|
|
|
+ let miniseed = 45;
|
|
|
+ let conv_id = b"XYZZYXYZ";
|
|
|
+
|
|
|
+ let mut inputs_iter = vec![b"\x00\xff%!)(*(@)_@(#)*_@!)%*(".to_vec(),
|
|
|
+ b"barfoo".to_vec()].into_iter();
|
|
|
+ let key = GenericArray::from_slice(&[0u8; 16]);
|
|
|
+
|
|
|
+ let mut p = Packetizer{
|
|
|
+ rng: rand_chacha::ChaCha8Rng::seed_from_u64(miniseed),
|
|
|
+ aes_key: *key,
|
|
|
+ conversation_id: conv_id,
|
|
|
+ messages_iter: Box::new(&mut inputs_iter),
|
|
|
+ sequence_no: 0
|
|
|
+ };
|
|
|
+
|
|
|
+ let nx = p.next();
|
|
|
+ assert_matches!(nx, Some(..));
|
|
|
+
|
|
|
+ if let Some(stuff) = nx {
|
|
|
+ println!("{:?}", &stuff[..]);
|
|
|
+
|
|
|
+ let mx = try_read_packet_header(key, &stuff[..]);
|
|
|
+ assert_matches!(mx, Ok(..));
|
|
|
+
|
|
|
+ let mx = try_depacketize(key, &stuff[..]);
|
|
|
+ assert_matches!(mx, Ok(..));
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ #[test]
|
|
|
fn bigrationals_and_bytevecs() {
|
|
|
for input in vec![
|
|
|
vec![0x00, 0x00],
|
