Jsonrpc: add new tests using Python subprocesses

Most of these tests are for the scontrol/"anxious continuation"
mechanism (bug#80623)

The new ERT tests use Python subprocesses via stdin/stdout pipe as
JSONRPC endpoints.  A shared framing library lives in
jsonrpc-resources/common.py.

* test/lisp/jsonrpc-tests.el (jsonrpc--test-dir): New constant.
(jsonrpc--with-python-fixture): New macro.
(scontrol-remote-during-sync-1)
(scontrol-remote-during-sync-2)
(scontrol-anxious-nested)
(scontrol-remote-error)
(shutdown-clean-after-notification): New tests.

* test/lisp/jsonrpc-resources/common.py: New file.

* test/lisp/jsonrpc-resources/server-remote-during-sync-1.py: New file.

* test/lisp/jsonrpc-resources/server-remote-during-sync-2.py: New file.

* test/lisp/jsonrpc-resources/server-anxious-nested.py: New file.

* test/lisp/jsonrpc-resources/server-remote-error.py: New file.

* test/lisp/jsonrpc-resources/server-harakiri.py: New file.

7 files changed, 375 insertions, 0 deletions

diff --git a/test/lisp/jsonrpc-resources/common.py b/test/lisp/jsonrpc-resources/common.py
new file mode 100644
index 00000000000..6bcc1db8bba
--- /dev/null
+++ b/test/lisp/jsonrpc-resources/common.py
@@ -0,0 +1,32 @@
+"""Common JSONRPC framing helpers for jsonrpc.el test servers."""
+
+import json
+import sys
+
+def read_msg():
+    """Read one Content-Length-framed JSON-RPC message from stdin."""
+    headers = {}
+    while True:
+        line = sys.stdin.buffer.readline()
+        if not line:
+            return None
+        text = line.decode('utf-8').rstrip('\r\n')
+        if not text:
+            break
+        if ':' in text:
+            k, _, v = text.partition(':')
+            headers[k.strip()] = v.strip()
+    n = int(headers.get('Content-Length', 0))
+    return json.loads(sys.stdin.buffer.read(n).decode('utf-8')) if n else None
+
+def write_msg(msg):
+    """Write one Content-Length-framed JSON-RPC message to stdout."""
+    body = json.dumps(msg, ensure_ascii=False).encode('utf-8')
+    sys.stdout.buffer.write(
+        f'Content-Length: {len(body)}\r\n\r\n'.encode('utf-8') + body
+    )
+    sys.stdout.buffer.flush()
+
+def log(text):
+    """Write a log line to stderr."""
+    print(f'[test-server] {text}', file=sys.stderr, flush=True)
diff --git a/test/lisp/jsonrpc-resources/server-anxious-nested.py b/test/lisp/jsonrpc-resources/server-anxious-nested.py
new file mode 100755
index 00000000000..8169b0936d1
--- /dev/null
+++ b/test/lisp/jsonrpc-resources/server-anxious-nested.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+"""Test server for scontrol-anxious-nested.
+
+Choreography (exercises the anxious-continuation mechanism):
+
+  client -> server:  LR1              (id=1)
+  server -> client:  RR1              (id=1000)
+  server -> client:  response LR1     "lr1-ok"
+  client -> server:  LR2              (id=2)
+  server -> client:  response LR2     "lr2-ok"
+  client -> server:  response RR1     "lr2-ok"
+
+LR2 should complete first, then RR1, then LR1.
+"""
+import os
+import sys
+sys.path.insert(0, os.path.dirname(__file__))
+from common import read_msg, write_msg, log
+
+
+def main():
+    while True:
+        lr1 = read_msg()
+        if lr1 is None:
+            break
+        mid = lr1.get('id')
+        method = lr1.get('method')
+        log(f'<- {method or "(response)"} id={mid}')
+        if method == 'harakiri':
+            log('-> very clean harakiri')
+            break
+        elif method == 'LR1':
+            # Send RR1, then immediately respond to LR1 without awaiting
+            # anything.  The response-to-LR1 will be queued as anxious on the
+            # client while its rdispatcher blocks waiting for LR2.
+            write_msg({'jsonrpc': '2.0', 'id': 1000,
+                       'method': 'RR1', 'params': {}})
+            log('-> RR1 id=1000')
+            write_msg({'jsonrpc': '2.0', 'id': mid, 'result': 'lr1-ok'})
+            log(f'-> (response LR1) id={mid}')
+            # LR2 arrives next: the client's rdispatcher for RR1
+            # issues it as a nested sync request.
+            lr2 = read_msg()
+            fid = lr2.get('id') if lr2 else None
+            log(f'<- LR2 id={fid}')
+            write_msg({'jsonrpc': '2.0', 'id': fid, 'result': 'lr2-ok'})
+            log(f'-> (response LR2) id={fid}')
+            # Finally collect the RR1 response (rdispatcher return value).
+            rr1_resp = read_msg()
+            log(f'<- (response RR1) id={rr1_resp.get("id") if rr1_resp else None}')
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/lisp/jsonrpc-resources/server-harakiri.py b/test/lisp/jsonrpc-resources/server-harakiri.py
new file mode 100755
index 00000000000..c20a3fbdaee
--- /dev/null
+++ b/test/lisp/jsonrpc-resources/server-harakiri.py
@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+"""Test server for shutdown-clean test.
+
+Waits for a 'harakiri' notification and then exits cleanly.
+"""
+import os, sys
+sys.path.insert(0, os.path.dirname(__file__))
+from common import read_msg, log
+
+
+def main():
+    while True:
+        msg = read_msg()
+        if msg is None:
+            break
+        method = msg.get('method')
+        log(f'<- {method or "(response)"} id={msg.get("id")}')
+        if method == 'harakiri':
+            log('-> very clean harakiri')
+            break
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/lisp/jsonrpc-resources/server-remote-during-sync-1.py b/test/lisp/jsonrpc-resources/server-remote-during-sync-1.py
new file mode 100755
index 00000000000..b8cb549bbd5
--- /dev/null
+++ b/test/lisp/jsonrpc-resources/server-remote-during-sync-1.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+"""Test server for scontrol-remote-during-sync-1.
+
+Choreography (tests bug#80623):
+
+  client -> server:  LR1            (id=1)
+  server -> client:  RR1            (id=1000)
+  server -> client:  response LR1   "lr1-ok"
+  client -> server:  response RR1   "rr1-ok"
+"""
+import os
+import sys
+sys.path.insert(0, os.path.dirname(__file__))
+from common import read_msg, write_msg, log
+
+
+def main():
+    while True:
+        msg = read_msg()
+        if msg is None:
+            break
+        mid = msg.get('id')
+        method = msg.get('method')
+        log(f'<- {method or "(response)"} id={mid}')
+        if method == 'harakiri':
+            log('-> very clean harakiri')
+            break
+        elif method == 'LR1':
+            # Send RR1 request
+            write_msg({'jsonrpc': '2.0', 'id': 1000,
+                       'method': 'RR1', 'params': {}})
+            log('-> RR1 id=1000')
+            # Immediately reply to LR1
+            write_msg({'jsonrpc': '2.0', 'id': mid, 'result': 'lr1-ok'})
+            log(f'-> (response LR1) id={mid}')
+            # Now wait for reply to RR1
+            resp = read_msg()
+            log(f'<- (response RR1) id={resp.get("id") if resp else None}')
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/lisp/jsonrpc-resources/server-remote-during-sync-2.py b/test/lisp/jsonrpc-resources/server-remote-during-sync-2.py
new file mode 100755
index 00000000000..e6250a0553b
--- /dev/null
+++ b/test/lisp/jsonrpc-resources/server-remote-during-sync-2.py
@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+"""Test server for scontrol-remote-during-sync-2.
+
+Choreography (tests bug#80623):
+
+  client -> server:  LR1            (id=1)
+  server -> client:  RR1            (id=1000)
+  client -> server:  response RR1   "rr1-ok"
+  server -> client:  response LR1   "lr1-ok"
+"""
+import os
+import sys
+sys.path.insert(0, os.path.dirname(__file__))
+from common import read_msg, write_msg, log
+
+
+def main():
+    while True:
+        msg = read_msg()
+        if msg is None:
+            break
+        mid = msg.get('id')
+        method = msg.get('method')
+        log(f'<- {method or "(response)"} id={mid}')
+        if method == 'harakiri':
+            log('-> very clean harakiri')
+            break
+        elif method == 'LR1':
+            # Send RR1 request
+            write_msg({'jsonrpc': '2.0', 'id': 1000,
+                       'method': 'RR1', 'params': {}})
+            log('-> RR1 id=1000')
+            # Wait for reply to RR1
+            resp = read_msg()
+            log(f'<- (response RR1) id={resp.get("id") if resp else None}')
+            # Only NOW answer LR1
+            write_msg({'jsonrpc': '2.0', 'id': mid, 'result': 'lr1-ok'})
+            log(f'-> (response LR1) id={mid}')
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/lisp/jsonrpc-resources/server-remote-error.py b/test/lisp/jsonrpc-resources/server-remote-error.py
new file mode 100755
index 00000000000..00c907e189b
--- /dev/null
+++ b/test/lisp/jsonrpc-resources/server-remote-error.py
@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+"""Test server for scontrol-remote-error.
+
+Choreography (anxious continuation survives an rdispatcher error):
+
+  client -> server:  LR1               (id=1)
+  server -> client:  badMethod           (id=1000)
+  server -> client:  response LR1      "ok"
+  client -> server:  error response badMethod {code: -32601}
+
+Even though the remote-request dispatch produces an error reply, the
+anxious continuation for LR1 must still fire and resolve to "ok".
+"""
+import os, sys
+sys.path.insert(0, os.path.dirname(__file__))
+from common import read_msg, write_msg, log
+
+
+def main():
+    while True:
+        msg = read_msg()
+        if msg is None:
+            break
+        mid = msg.get('id')
+        method = msg.get('method')
+        log(f'<- {method or "(response)"} id={mid}')
+        if method == 'harakiri':
+            log('-> very clean harakiri')
+            break
+        elif method == 'LR1':
+            # Send badMethod BEFORE responding to LR1; the client
+            # rdispatcher will signal a jsonrpc-error for it.
+            write_msg({'jsonrpc': '2.0', 'id': 1000,
+                       'method': 'badMethod', 'params': {}})
+            log('-> badMethod id=1000')
+            write_msg({'jsonrpc': '2.0', 'id': mid, 'result': 'ok'})
+            log(f'-> (response LR1) id={mid}')
+            # Collect the error response to badMethod.
+            err = read_msg()
+            log(f'<- (error response badMethod): {err}')
+
+
+if __name__ == '__main__':
+    main()
diff --git a/test/lisp/jsonrpc-tests.el b/test/lisp/jsonrpc-tests.el
index ca26f3c5a30..cdb4e04fc39 100644
--- a/test/lisp/jsonrpc-tests.el
+++ b/test/lisp/jsonrpc-tests.el
@@ -252,5 +252,142 @@
       (should (eq 2 n-deferred-2))
       (should (eq 0 (hash-table-count (jsonrpc--deferred-actions conn)))))))
 
+
+;;; Tests using Python subprocesses (scontrol / anxious mechanism)
+;;;
+
+(defconst jsonrpc--test-dir
+  (file-name-directory (or load-file-name buffer-file-name))
+  "Directory of this test file, captured at load time.")
+
+(cl-defmacro jsonrpc--with-python-fixture ((script conn &rest initargs) &body body)
+  "Start SCRIPT under python3 as a pipe subprocess, bind connection to CONN.
+SCRIPT is a path relative to this file's directory.
+INITARGS are passed to `make-instance' for `jsonrpc-process-connection'."
+  (declare (indent 1))
+  `(let ((,conn nil))
+     (unwind-protect
+         (progn
+           (setq ,conn
+                 (make-instance
+                  'jsonrpc-process-connection
+                  :name "jsonrpc-python-test"
+                  :process (make-process
+                             :name "jsonrpc-python-test"
+                             :command (list "python3"
+                                            (expand-file-name
+                                             ,script
+                                             jsonrpc--test-dir))
+                             :connection-type 'pipe
+                             :noquery t)
+                  ,@initargs))
+           (with-timeout (5
+                          (when ,conn
+                            (let ((buf (jsonrpc--events-buffer ,conn)))
+                              (when (buffer-live-p buf)
+                                (if noninteractive
+                                    (progn
+                                      (message "contents of `%s':" (buffer-name buf))
+                                      (princ (with-current-buffer buf (buffer-string))
+                                             #'external-debugging-output))
+                                  (message "Preserved for inspection: %s"
+                                           (buffer-name buf))))))
+                          (ert-fail "Test timed out after 5s"))
+             ,@body))
+       (when ,conn
+         (ignore-errors
+          (jsonrpc-notify ,conn 'harakiri nil)
+          (kill-buffer (jsonrpc--events-buffer ,conn))
+          (jsonrpc-shutdown ,conn))))))
+
+(ert-deftest scontrol-remote-during-sync-1 ()
+  "Anxious local continuations.
+Endpoint sends a remote request RR1 on LR1, then replies to LR1
+immediately before waiting for RR1 to resolve.
+This is what JETLS does (bug#80623)."
+  (skip-unless (executable-find "python3"))
+  (skip-when (eq system-type 'windows-nt))
+  (jsonrpc--with-python-fixture
+      ("jsonrpc-resources/server-remote-during-sync-1.py" conn
+       :request-dispatcher
+       (lambda (_conn method _params)
+         (pcase method
+           ('RR1 "rr1-ok")
+           (_ (error "unexpected method: %s" method)))))
+    (should (equal "lr1-ok" (jsonrpc-request conn 'LR1 [] :timeout 5)))))
+
+(ert-deftest scontrol-remote-during-sync-2 ()
+  "Anxious local continuations.
+Exactly the same test as 2, but different endpoint, which now still
+sends RR1 on LR1 but now waits for RR1 to resolve before replying to
+LR1.
+This is what GoPls does (bug#80623)."
+  (skip-unless (executable-find "python3"))
+  (skip-when (eq system-type 'windows-nt))
+  (jsonrpc--with-python-fixture
+      ("jsonrpc-resources/server-remote-during-sync-2.py" conn
+       :request-dispatcher
+       (lambda (_conn method _params)
+         (pcase method
+           ('RR1 "rr1-ok")
+           (_ (error "unexpected method: %s" method)))))
+    (should (equal "lr1-ok" (jsonrpc-request conn 'LR1 [] :timeout 5)))))
+
+(ert-deftest scontrol-anxious-nested ()
+  "Nested anxious continuations
+Two local sync requests LR1 and LR2 with a remote RR1 in between.
+Vaguely similar to Julia's JETLS (bug#80623), but more complex."
+  (skip-unless (executable-find "python3"))
+  (skip-when (eq system-type 'windows-nt))
+  (let (lr2-result completed)
+    (jsonrpc--with-python-fixture
+     ("jsonrpc-resources/server-anxious-nested.py" conn
+      :request-dispatcher
+      (lambda (conn method _params)
+        (pcase method
+          ('RR1
+           (setq lr2-result
+                 (jsonrpc-request conn 'LR2 [] :timeout 5))
+           (push "lr2" completed)
+           (push "rr1" completed))
+          (_ (error "unexpected method: %s" method)))))
+     (should (equal "lr1-ok" (jsonrpc-request conn 'LR1 [] :timeout 5)))
+     (push "lr1" completed)
+     (should (equal "lr2-ok" lr2-result))
+     (should (equal '("lr1" "rr1" "lr2") completed)))))
+
+(ert-deftest scontrol-remote-error ()
+  "Anxious continuation even when rdispatcher signals errors."
+  (skip-unless (executable-find "python3"))
+  (skip-when (eq system-type 'windows-nt))
+  (jsonrpc--with-python-fixture
+      ("jsonrpc-resources/server-remote-error.py" conn
+       :request-dispatcher
+       (lambda (_conn method _params)
+         (pcase method
+           ('badMethod
+            (signal 'jsonrpc-error
+                    '((jsonrpc-error-message . "method not allowed")
+                      (jsonrpc-error-code . -32601))))
+           (_ (error "unexpected method: %s" method)))))
+    (should (equal "ok" (jsonrpc-request conn 'LR1 [] :timeout 5)))))
+
+(ert-deftest shutdown-clean-after-notification ()
+  "Server exits cleanly after harakiri notification.
+`jsonrpc-shutdown' should not emit a \"Sentinel hasn't run\" warning."
+  (skip-unless (executable-find "python3"))
+  (skip-when (eq system-type 'windows-nt))
+  (let (warned)
+    (cl-letf (((symbol-function 'jsonrpc--warn)
+               (lambda (fmt &rest args)
+                 (setq warned (apply #'format fmt args)))))
+      (jsonrpc--with-python-fixture
+          ("jsonrpc-resources/server-harakiri.py" conn)
+        (jsonrpc-notify conn 'harakiri nil)
+        ;; Give the server time to exit before shutdown checks the sentinel.
+        (accept-process-output nil 0.3)
+        (jsonrpc-shutdown conn)))
+    (should-not warned)))
+
 (provide 'jsonrpc-tests)
 ;;; jsonrpc-tests.el ends here