Refactoring in preparation for WebSockets (#254)

* Fix injector race conditions

* add some more tasks

* hide useless rollup warnings

* goodbye to clientsession errors

* completely fix desktop mode switch race condition

* fix typos and TS warning in plugin error handler

* fix chown error

* start debugger if needed and not already started

* fix get_steam_resource for the like 2 legacy plugins still using it lol

* add ClientOSError to get_tabs error handling

.vscode/tasks.json

@@ -14,7 +14,9 @@
             "label": "localrun",
             "type": "shell",
             "group": "none",
-            "dependsOn" : ["buildall"],
+            "dependsOn": [
+                "buildall"
+            ],
             "detail": "Check for local runs, create a plugins folder",
             "command": "mkdir -p plugins",
             "problemMatcher": []
@@ -48,6 +50,16 @@
             "command": "cd frontend && pnpm i",
             "problemMatcher": []
         },
+        {
+            "script": "watch",
+            "type": "npm",
+            "path": "frontend",
+            "group": "build",
+            "problemMatcher": [],
+            "label": "watchfrontend",
+            "detail": "rollup -c -w",
+            "isBackground": true
+        },
         {
             "label": "buildfrontend",
             "type": "npm",
@@ -55,8 +67,7 @@
             "detail": "rollup -c",
             "script": "build",
             "path": "frontend",
-            "problemMatcher": [],
+            "problemMatcher": []
-            
         },
         {
             "label": "buildall",
@@ -95,7 +106,9 @@
             "detail": "Run indev PluginLoader on Deck",
             "type": "shell",
             "group": "none",
-            "dependsOn" : ["checkforsettings"],
+            "dependsOn": [
+                "checkforsettings"
+            ],
             "command": "ssh deck@${config:deckip} -p ${config:deckport} ${config:deckkey} 'export PLUGIN_PATH=${config:deckdir}/homebrew/dev/plugins; export CHOWN_PLUGIN_PATH=0; export LOG_LEVEL=DEBUG; cd ${config:deckdir}/homebrew/services; echo '${config:deckpass}' | sudo -SE python3 ${config:deckdir}/homebrew/dev/pluginloader/backend/main.py'",
             "problemMatcher": []
         },
@@ -108,6 +121,20 @@
             "problemMatcher": []
         },
         // ALL-IN-ONES
+        {
+            "label": "deployandrun",
+            "detail": "Deploy and run, skipping JS build. Useful when combined with npm:watch",
+            "dependsOrder": "sequence",
+            "group": {
+                "kind": "build",
+                "isDefault": true
+            },
+            "dependsOn": [
+                "deploy",
+                "runpydeck"
+            ],
+            "problemMatcher": []
+        },
         {
             "label": "updateremote",
             "detail": "Build and deploy",
@@ -115,7 +142,7 @@
             "group": "none",
             "dependsOn": [
                 "buildall",
-                "deploy",
+                "deploy"
             ],
             "problemMatcher": []
         },

backend/injector.py

@@ -6,7 +6,7 @@ from traceback import format_exc
 from typing import List
 
 from aiohttp import ClientSession, WSMsgType
-from aiohttp.client_exceptions import ClientConnectorError
+from aiohttp.client_exceptions import ClientConnectorError, ClientOSError
 from asyncio.exceptions import TimeoutError
 import uuid
 
@@ -32,12 +32,15 @@ class Tab:
         self.websocket = await self.client.ws_connect(self.ws_url)
 
     async def close_websocket(self):
+        await self.websocket.close()
         await self.client.close()
 
     async def listen_for_message(self):
         async for message in self.websocket:
             data = message.json()
             yield data
+        logger.warn(f"The Tab {self.title} socket has been disconnected while listening for messages.")
+        await self.close_websocket()
             
     async def _send_devtools_cmd(self, dc, receive=True):
         if self.websocket:
@@ -52,6 +55,7 @@ class Tab:
         raise RuntimeError("Websocket not opened")
 
     async def evaluate_js(self, js, run_async=False, manage_socket=True, get_result=True):
+        try:
             if manage_socket:
                 await self.open_websocket()
 
@@ -64,6 +68,7 @@ class Tab:
                 }
             }, get_result)
 
+        finally:
             if manage_socket:
                 await self.close_websocket()
         return res
@@ -77,6 +82,7 @@ class Tab:
         return res["result"]["result"]["value"]
 
     async def close(self, manage_socket=True):
+        try:
             if manage_socket:
                 await self.open_websocket()
 
@@ -84,6 +90,7 @@ class Tab:
                 "method": "Page.close",
             }, False)
 
+        finally:
             if manage_socket:
                 await self.close_websocket()
         return res
@@ -105,6 +112,7 @@ class Tab:
         }, False)
 
     async def refresh(self):
+        try:
             if manage_socket:
                 await self.open_websocket()
 
@@ -112,6 +120,7 @@ class Tab:
                 "method": "Page.reload",
             }, False)
 
+        finally:
             if manage_socket:
                 await self.close_websocket()
 
@@ -120,6 +129,7 @@ class Tab:
         """
         Reloads the current tab, with JS to run on load via debugger
         """
+        try:
             if manage_socket:
                 await self.open_websocket()
 
@@ -175,6 +185,7 @@ class Tab:
                 "method": "Debugger.disable"
             }, True)
 
+        finally:
             if manage_socket:
                 await self.close_websocket()
         return
@@ -212,6 +223,7 @@ class Tab:
             (see remove_script_to_evaluate_on_new_document below)
             None is returned if `get_result` is False
         """
+        try:
 
             wrappedjs = """
             function scriptFunc() {
@@ -236,6 +248,7 @@ class Tab:
                 }
             }, get_result)
 
+        finally:
             if manage_socket:
                 await self.close_websocket()
         return res
@@ -250,6 +263,7 @@ class Tab:
             The identifier of the script to remove (returned from `add_script_to_evaluate_on_new_document`)
         """
 
+        try:
             if manage_socket:
                 await self.open_websocket()
 
@@ -260,6 +274,7 @@ class Tab:
                 }
             }, False)
 
+        finally:
             if manage_socket:
                 await self.close_websocket()
 
@@ -337,17 +352,21 @@ class Tab:
 
 
 async def get_tabs() -> List[Tab]:
-    async with ClientSession() as web:
     res = {}
 
+    na = False
     while True:
         try:
+            async with ClientSession() as web:
                 res = await web.get(f"{BASE_ADDRESS}/json", timeout=3)
         except ClientConnectorError:
-                logger.debug("ClientConnectorError excepted.")
+            if not na:
                 logger.debug("Steam isn't available yet. Wait for a moment...")
-                logger.error(format_exc())
+                na = True
             await sleep(5)
+        except ClientOSError:
+            logger.warn(f"The request to {BASE_ADDRESS}/json was reset")
+            await sleep(1)
         except TimeoutError:
             logger.warn(f"The request to {BASE_ADDRESS}/json timed out")
             await sleep(1)

backend/loader.py

@@ -208,7 +208,7 @@ class Loader:
         return web.Response(text=ret)
 
     async def get_steam_resource(self, request):
-        tab = await get_tab("QuickAccess")
+        tab = await get_tab("SP")
         try:
             return web.Response(text=await tab.get_steam_resource(f"https://steamloopback.host/{request.match_info['path']}"), content_type="text/html")
         except Exception as e:

backend/main.py

@@ -4,7 +4,7 @@ from subprocess import call
 if hasattr(sys, '_MEIPASS'):
     call(['chmod', '-R', '755', sys._MEIPASS])
 # Full imports
-from asyncio import get_event_loop, sleep
+from asyncio import new_event_loop, set_event_loop, sleep
 from json import dumps, loads
 from logging import DEBUG, INFO, basicConfig, getLogger
 from os import getenv, chmod, path
@@ -12,7 +12,7 @@ from traceback import format_exc
 
 import aiohttp_cors
 # Partial imports
-from aiohttp import ClientSession, client_exceptions, WSMsgType
+from aiohttp import client_exceptions, WSMsgType
 from aiohttp.web import Application, Response, get, run_app, static
 from aiohttp_jinja2 import setup as jinja_setup
 
@@ -21,7 +21,7 @@ from browser import PluginBrowser
 from helpers import (REMOTE_DEBUGGER_UNIT, csrf_middleware, get_csrf_token,
                      get_home_path, get_homebrew_path, get_user,
                      get_user_group, set_user, set_user_group,
-                     stop_systemd_unit)
+                     stop_systemd_unit, start_systemd_unit)
 from injector import get_gamepadui_tab, Tab, get_tabs
 from loader import Loader
 from settings import SettingsManager
@@ -56,15 +56,15 @@ basicConfig(
 
 logger = getLogger("Main")
 
-async def chown_plugin_dir(_):
+async def chown_plugin_dir():
     code_chown = call(["chown", "-R", USER+":"+GROUP, CONFIG["plugin_path"]])
     code_chmod = call(["chmod", "-R", "555", CONFIG["plugin_path"]])
     if code_chown != 0 or code_chmod != 0:
         logger.error(f"chown/chmod exited with a non-zero exit code (chown: {code_chown}, chmod: {code_chmod})")
 
 class PluginManager:
-    def __init__(self) -> None:
+    def __init__(self, loop) -> None:
-        self.loop = get_event_loop()
+        self.loop = loop
         self.web_app = Application()
         self.web_app.middlewares.append(csrf_middleware)
         self.cors = aiohttp_cors.setup(self.web_app, defaults={
@@ -81,12 +81,19 @@ class PluginManager:
         self.updater = Updater(self)
 
         jinja_setup(self.web_app)
+
+        async def startup(_):
+            if self.settings.getSetting("cef_forward", False):
+                self.loop.create_task(start_systemd_unit(REMOTE_DEBUGGER_UNIT))
+            else:
+                self.loop.create_task(stop_systemd_unit(REMOTE_DEBUGGER_UNIT))
             if CONFIG["chown_plugin_path"] == True:
-            self.web_app.on_startup.append(chown_plugin_dir)
+                chown_plugin_dir()
             self.loop.create_task(self.loader_reinjector())
             self.loop.create_task(self.load_plugins())
-        if not self.settings.getSetting("cef_forward", False):
+
-            self.loop.create_task(stop_systemd_unit(REMOTE_DEBUGGER_UNIT))
+        self.web_app.on_startup.append(startup)
+
         self.loop.set_exception_handler(self.exception_handler)
         self.web_app.add_routes([get("/auth/token", self.get_auth_token)])
 
@@ -103,31 +110,29 @@ class PluginManager:
     async def get_auth_token(self, request):
         return Response(text=get_csrf_token())
 
-    async def wait_for_server(self):
-        async with ClientSession() as web:
-            while True:
-                try:
-                    await web.get(f"http://{CONFIG['server_host']}:{CONFIG['server_port']}")
-                    return
-                except Exception as e:
-                    await sleep(0.1)
-
     async def load_plugins(self):
-        await self.wait_for_server()
+        # await self.wait_for_server()
+        logger.debug("Loading plugins")
         self.plugin_loader.import_plugins()
         # await inject_to_tab("SP", "window.syncDeckyPlugins();")
 
     async def loader_reinjector(self):
         while True:
             tab = None
+            nf = False
+            dc = False
             while not tab:
                 try:
                     tab = await get_gamepadui_tab()
                 except client_exceptions.ClientConnectorError or client_exceptions.ServerDisconnectedError:
-                    logger.debug("Couldn't connect to debugger, waiting 5 seconds.")
+                    if not dc:
+                        logger.debug("Couldn't connect to debugger, waiting...")
+                        dc = True
                     pass
                 except ValueError:
-                    logger.debug("Couldn't find GamepadUI tab, waiting 5 seconds")
+                    if not nf:
+                        logger.debug("Couldn't find GamepadUI tab, waiting...")
+                        nf = True
                     pass
                 if not tab:
                     await sleep(5)
@@ -136,15 +141,20 @@ class PluginManager:
             await self.inject_javascript(tab, True)
             try:
                 async for msg in tab.listen_for_message():
+                    # this gets spammed a lot
+                    if msg.get("method", None) != "Page.navigatedWithinDocument":
                         logger.debug("Page event: " + str(msg.get("method", None)))
                         if msg.get("method", None) == "Page.domContentEventFired":
                             if not await tab.has_global_var("deckyHasLoaded", False):
                                 await self.inject_javascript(tab)
-                    if msg.get("method", None) == "Inspector.detached" or msg.get("type", None) in (WSMsgType.CLOSED, WSMsgType.ERROR):
+                        if msg.get("method", None) == "Inspector.detached":
-                        logger.info("CEF has disconnected...")
+                            logger.info("CEF has requested that we detach.")
-                        logger.debug("Exit message: " + str(msg))
                             await tab.close_websocket()
                             break
+                # If this is a forceful disconnect the loop will just stop without any failure message. In this case, injector.py will handle this for us so we don't need to close the socket.
+                # This is because of https://github.com/aio-libs/aiohttp/blob/3ee7091b40a1bc58a8d7846e7878a77640e96996/aiohttp/client_ws.py#L321
+                logger.info("CEF has disconnected...")
+                # At this point the loop starts again and we connect to the freshly started Steam client once it is ready.
             except Exception as e:
                 logger.error("Exception while reading page events " + format_exc())
                 await tab.close_websocket()
@@ -166,7 +176,7 @@ class PluginManager:
             #                 logger.debug("Closing tab: " + getattr(t, "title", "Untitled"))
             #                 await t.close()
             #                 await sleep(0.5)
-            await tab.evaluate_js("try{if (window.deckyHasLoaded){setTimeout(() => SteamClient.User.StartRestart(), 100)}else{window.deckyHasLoaded = true;(async()=>{while(!window.SP_REACT){await new Promise(r => setTimeout(r, 10))};await import('http://localhost:1337/frontend/index.js')})();}}catch(e){console.error(e)}", False, False, False)
+            await tab.evaluate_js("try{if (window.deckyHasLoaded){setTimeout(() => SteamClient.User.StartRestart(), 100)}else{window.deckyHasLoaded = true;(async()=>{try{while(!window.SP_REACT){await new Promise(r => setTimeout(r, 10))};await import('http://localhost:1337/frontend/index.js')}catch(e){console.error(e)};})();}}catch(e){console.error(e)}", False, False, False)
         except:
             logger.info("Failed to inject JavaScript into tab\n" + format_exc())
             pass
@@ -175,4 +185,6 @@ class PluginManager:
         return run_app(self.web_app, host=CONFIG["server_host"], port=CONFIG["server_port"], loop=self.loop, access_log=None)
 
 if __name__ == "__main__":
-    PluginManager().run()
+    loop = new_event_loop()
+    set_event_loop(loop)
+    PluginManager(loop).run()

backend/updater.py

@@ -198,7 +198,7 @@ class Updater:
             logger.info("Updated loader installation.")
             await tab.evaluate_js("window.DeckyUpdater.finish()", False, False)
             await self.do_restart()
-                await tab.client.close()
+            await tab.close_websocket()
 
     async def do_restart(self):
         call(["systemctl", "daemon-reload"])

backend/utilities.py

@@ -80,7 +80,7 @@ class Utilities:
 
     async def http_request(self, method="", url="", **kwargs):
         async with ClientSession() as web:
-            async with web.request(method, url, ssl=helpers.get_ssl_context(), **kwargs) as res:
+            res = await web.request(method, url, ssl=helpers.get_ssl_context(), **kwargs)
         return {
             "status": res.status,
             "headers": dict(res.headers),
@@ -241,7 +241,7 @@ class Utilities:
             if ip != None:
                 self.logger.info("Connecting to React DevTools at " + ip)
                 async with ClientSession() as web:
-                    async with web.request("GET", "http://" + ip + ":8097", ssl=helpers.get_ssl_context()) as res:
+                    res = await web.request("GET", "http://" + ip + ":8097", ssl=helpers.get_ssl_context())
                 if res.status != 200:
                     self.logger.error("Failed to connect to React DevTools at " + ip)
                     return False

frontend/rollup.config.js

@@ -5,7 +5,12 @@ import externalGlobals from "rollup-plugin-external-globals";
 import del from 'rollup-plugin-delete'
 import replace from '@rollup/plugin-replace';
 import typescript from '@rollup/plugin-typescript';
-import { defineConfig } from 'rollup';
+import { defineConfig, handleWarning } from 'rollup';
+
+const hiddenWarnings = [
+ "THIS_IS_UNDEFINED",
+ "EVAL"
+];
 
 export default defineConfig({
   input: 'src/index.tsx',
@@ -35,5 +40,9 @@ export default defineConfig({
     chunkFileNames: (chunkInfo) => {
       return 'chunk-[hash].js'
     }
+  },
+  onwarn: function ( message ) {
+    if (hiddenWarnings.some(warning => message.code === warning)) return;
+    handleWarning(message);
   }
 });

frontend/src/plugin-loader.tsx

@@ -249,11 +249,11 @@ class PluginLoader extends Logger {
           <>
             Error:{' '}
             <pre>
-              <code>{e instanceof Error ? e.stack : e?.toString()}</code>
+              <code>{e instanceof Error ? e.stack : JSON.stringify(e)}</code>
             </pre>
             <>
-              Please go to <FaCog style={{ display: 'inline' }} /> in Decky Loader.e settings menu if you need to
+              Please go to <FaCog style={{ display: 'inline' }} /> in the Decky menu if you need to uninstall this
-              uninstall this plugin.
+              plugin.
             </>
           </>
         );