use utf-16 for everything because of javascript

Cargo.lock

@@ -1084,6 +1084,12 @@ dependencies = [
  "tinyvec",
 ]
 
+[[package]]
+name = "unicode-segmentation"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1dd624098567895118886609431a7c3b8f516e41d30e0643f03d94592a147e36"
+
 [[package]]
 name = "url"
 version = "2.3.1"
@@ -1246,4 +1252,5 @@ dependencies = [
  "tower-http",
  "tracing",
  "tracing-subscriber",
+ "unicode-segmentation",
 ]

Cargo.toml

@@ -14,3 +14,4 @@ tokio = { version = "1.25.0", features = ["full"] }
 tower-http = { version = "0.3.5", features = ["trace"] }
 tracing = "0.1.37"
 tracing-subscriber = { version = "0.3.16", features = ["env-filter"] }
+unicode-segmentation = "1.10.1"

assets/index.js

@@ -1,10 +1,10 @@
 const areaId = "doctext";
 const validEvents = [
-  "insertText",
+    "insertText",
-  "insertFromPaste",
+    "insertFromPaste",
-  "insertLineBreak",
+    "insertLineBreak",
-  "deleteContentBackward",
+    "deleteContentBackward",
-  "deleteContentForward",
+    "deleteContentForward",
 ];
 
 let selectionStart = 0;
@@ -16,60 +16,71 @@ let ws;
 const uuid = self.crypto.randomUUID();
 const wsUrl = "ws://localhost:3000/ws";
 
-function setup() {
+const textEncoder = new TextEncoder();
-  area = document.querySelector(`#${areaId}`);
+const textDecoder = new TextDecoder();
-  ws = new WebSocket(wsUrl);
 
-  setupUi();
+function setup() {
-  setupWs();
+    area = document.querySelector(`#${areaId}`);
+    ws = new WebSocket(wsUrl);
+
+    setupUi();
+    setupWs();
 }
 
 function setupUi() {
-  document.addEventListener("selectionchange", onSelectionChange, false);
+    document.addEventListener("selectionchange", onSelectionChange, false);
-  area.addEventListener("input", onInput, false);
+    area.addEventListener("beforeinput", onInput, false);
 }
 
 function setupWs() {
-  ws.onclose = function (e) {
+    ws.onclose = function (e) {
-    console.log(e);
+        console.log(e);
-    setTimeout(() => {
+        setTimeout(() => {
-      ws = new WebSocket(wsUrl);
+            ws = new WebSocket(wsUrl);
-    }, 2000);
+        }, 2000);
-  };
+    };
 
-  ws.onmessage = function (e) {
+    ws.onmessage = function (e) {
-    let payload = JSON.parse(e.data);
+        let payload = JSON.parse(e.data);
-    if (payload.client !== uuid) {
+        if (payload.client !== uuid) {
-      area.value = payload.doc;
+            console.log(payload.doc);
-    }
+            area.value = payload.doc;
-  };
+        }
+    };
 }
 
 function onSelectionChange() {
-  const activeElement = document.activeElement;
+    const activeElement = document.activeElement;
 
-  if (activeElement && activeElement.id === areaId) {
+    if (activeElement && activeElement.id === areaId) {
-    selectionStart = area.selectionStart;
+        console.debug("onSelectionChange", area);
-    selectionEnd = area.selectionEnd;
+        selectionStart = area.selectionStart;
-  }
+        selectionEnd = area.selectionEnd;
+    }
 }
 
 function onInput(event) {
-  if (!validEvents.includes(event.inputType)) return;
+    if (!validEvents.includes(event.inputType)) return;
-  const payload = {
-    client: uuid,
-    action: event.inputType,
-    data: event.data,
-    start: selectionStart,
-    end: selectionEnd,
-  };
 
-  ws.send(JSON.stringify(payload));
+    // workaround for differences between firefox and chrome
+    // chrome does not fire selectionchange events on backspace/delete events,
+    // while firefox does
+    if (event.inputType === "deleteContentBackward") {
+        selectionStart = area.selectionStart;
+        selectionEnd = area.selectionEnd;
+    }
 
-  console.log(event.inputType);
+    const payload = {
-  console.log(selectionStart);
+        client: uuid,
-  console.log(selectionEnd);
+        action: event.inputType,
-  console.log(event.data);
+        data: event.data,
+        start: selectionStart,
+        end: selectionEnd,
+    };
+
+    ws.send(JSON.stringify(payload));
+
+    console.log(selectionStart, selectionEnd);
 }
 
 setup();

src/action.rs

@@ -3,9 +3,7 @@ use std::sync::Arc;
 use serde::{Deserialize, Serialize};
 
 use crate::actions::delete::{DeleteContentBackward, DeleteContentForward};
-use crate::actions::insert::InsertText;
+use crate::actions::insert::{InsertText, InsertFromPaste, InsertLineBreak};
-use crate::actions::linebreak::InsertLineBreak;
-use crate::actions::paste::InsertFromPaste;
 use crate::actions::ActionRunner;
 use crate::AppState;
 
@@ -40,7 +38,11 @@ impl Action {
                 ActionType::DeleteContentForward => Box::new(DeleteContentForward),
             };
 
-            *doc = action.run(self.start, self.end, self.data.clone(), doc.clone());
+            let data = match self.data.to_owned() {
+                None => Vec::new(),
+                Some(data) => data.encode_utf16().collect()
+            };
+            *doc = action.run(self.start, self.end, data, doc.to_owned());
         }
     }
 }

src/actions/delete.rs

@@ -1,22 +1,45 @@
 use super::ActionRunner;
 
 pub(crate) struct DeleteContentBackward;
+
 pub(crate) struct DeleteContentForward;
 
-impl ActionRunner for DeleteContentBackward {
+fn delete(start: usize, end: usize, mut doc: Vec<u16>) -> Vec<u16> {
-    fn run(&self, start: usize, _end: usize, _data: Option<String>, mut doc: String) -> String {
+    if start > doc.len() + 1 {
-        if start > 0 {
+        return doc;
-            doc.remove(start - 1);
+    }
+
+    let mut start_idx = start;
+    let mut end_idx = end;
+    unsafe {
+        while start_idx > 0 && matches!(doc.get_unchecked(start_idx), 0xDC00..=0xDFFF) {
+            start_idx -= 1;
         }
-        doc
+
+        let end_byte = doc.get_unchecked(end_idx - 1);
+        if end_idx < doc.len() && end_byte > &0xD800 && end_byte < &0xDC00 {
+            end_idx += 1;
+        }
+    }
+
+    doc.drain(start_idx..end_idx);
+    doc
+}
+
+impl ActionRunner for DeleteContentBackward {
+    fn run(&self, start: usize, end: usize, _data: Vec<u16>, doc: Vec<u16>) -> Vec<u16> {
+        if (start == 0 && start == end ) || end > doc.len() { return doc; }
+        let (end, start) = if start == end { (start, start - 1) } else { (end, start) };
+
+        delete(start, end, doc)
     }
 }
 
 impl ActionRunner for DeleteContentForward {
-    fn run(&self, start: usize, _end: usize, _data: Option<String>, mut doc: String) -> String {
+    fn run(&self, start: usize, end: usize, _data: Vec<u16>, doc: Vec<u16>) -> Vec<u16> {
-        if doc.len() > start {
+        if start >= doc.len() { return doc; }
-            doc.remove(start);
+
-        }
+        let end = if start == end { start + 1 } else { end };
-        doc
+        delete(start, end, doc)
     }
 }

src/actions/insert.rs

@@ -1,18 +1,38 @@
 use super::ActionRunner;
 
 pub(crate) struct InsertText;
+pub(crate) struct InsertFromPaste;
+pub(crate) struct InsertLineBreak;
+
+const NL_BYTES: [u16; 1] = [10];
+
+fn insert(start: usize, end: usize, data: Vec<u16>, mut doc: Vec<u16>) -> Vec<u16> {
+    if doc.len() < start || data.is_empty() {
+        return doc;
+    }
+
+    if start < end {
+        doc.drain(start..end);
+    }
+
+    doc.splice(start..start, data);
+    doc
+}
 
 impl ActionRunner for InsertText {
-    fn run(&self, start: usize, end: usize, data: Option<String>, mut doc: String) -> String {
+    fn run(&self, start: usize, end: usize, data: Vec<u16>, doc: Vec<u16>) -> Vec<u16> {
-        if doc.len() < start || data.is_none() {
+        insert(start, end, data, doc)
-            return doc;
+    }
-        }
+}
-
+
-        if start < end {
+impl ActionRunner for InsertFromPaste {
-            doc.replace_range(start..end, "");
+    fn run(&self, start: usize, end: usize, data: Vec<u16>, doc: Vec<u16>) -> Vec<u16> {
-        }
+        insert(start, end, data, doc)
-
+    }
-        doc.insert_str(start, &data.unwrap());
+}
-        doc
+
+impl ActionRunner for InsertLineBreak {
+    fn run(&self, start: usize, _end: usize, _data: Vec<u16>, doc: Vec<u16>) -> Vec<u16> {
+        insert(start, start, Vec::from(NL_BYTES), doc)
     }
 }

src/actions/linebreak.rs

@@ -1,12 +0,0 @@
-use super::ActionRunner;
-
-pub(crate) struct InsertLineBreak;
-
-impl ActionRunner for InsertLineBreak {
-    fn run(&self, start: usize, _end: usize, _data: Option<String>, mut doc: String) -> String {
-        if doc.len() >= start {
-            doc.insert(start, '\n');
-        }
-        doc
-    }
-}

src/actions/mod.rs

@@ -1,8 +1,6 @@
 pub(crate) mod delete;
 pub(crate) mod insert;
-pub(crate) mod linebreak;
-pub(crate) mod paste;
 
 pub(crate) trait ActionRunner {
-    fn run(&self, start: usize, end: usize, data: Option<String>, doc: String) -> String;
+    fn run(&self, start: usize, end: usize, data: Vec<u16>, doc: Vec<u16>) -> Vec<u16>;
 }

src/actions/paste.rs

@@ -1,18 +0,0 @@
-use super::ActionRunner;
-
-pub(crate) struct InsertFromPaste;
-
-impl ActionRunner for InsertFromPaste {
-    fn run(&self, start: usize, end: usize, data: Option<String>, mut doc: String) -> String {
-        if doc.len() < start || data.is_none() {
-            return doc;
-        }
-
-        if start < end {
-            doc.replace_range(start..end, "");
-        }
-
-        doc.insert_str(start, &data.unwrap());
-        doc
-    }
-}

src/main.rs

@@ -13,7 +13,7 @@ mod actions;
 mod ws;
 
 struct AppState {
-    doc: RwLock<String>,
+    doc: RwLock<Vec<u16>>,
     tx: broadcast::Sender<String>,
 }
 
@@ -29,7 +29,7 @@ async fn main() {
 
     let (tx, _rx) = broadcast::channel(100);
     let app_state = Arc::new(AppState {
-        doc: RwLock::new("".to_string()),
+        doc: RwLock::new(Vec::new()),
         tx,
     });
 

src/ws.rs

@@ -31,7 +31,7 @@ async fn handle_socket(stream: WebSocket, state: Arc<AppState>) {
 
     {
         let doc = state.doc.read().await;
-        let doc: String = doc.clone();
+        let doc: String = String::from_utf16_lossy(&doc);
         let new_state = NewState {
             client: String::from("0000"),
             doc: doc.to_string(),
@@ -64,7 +64,7 @@ async fn handle_socket(stream: WebSocket, state: Arc<AppState>) {
 
                 let new_state = NewState {
                     client,
-                    doc: doc.to_string(),
+                    doc: String::from_utf16_lossy(&doc),
                 };
                 let payload = serde_json::to_string(&new_state).unwrap();
                 let _ = tx.send(payload);