implement fetch and external resource request apis

.vscode/defsettings.json

@@ -1,6 +1,7 @@
 {
     "deckip"    : "0.0.0.0",
     "deckport"  : "22",
+    "deckuser"  : "deck",
     "deckpass"  : "ssap",
     "deckkey"   : "-i ${env:HOME}/.ssh/id_rsa",
     "deckdir"   : "/home/deck"

.vscode/tasks.json

@@ -41,7 +41,7 @@
                 "deploy"
             ],
             "detail": "Check for local runs, create a plugins folder",
-            "command": "ssh deck@${config:deckip} -p ${config:deckport} ${config:deckkey} 'python -m ensurepip && python -m pip install --user --upgrade poetry && cd ${config:deckdir}/homebrew/dev/pluginloader/backend && python -m poetry install'",
+            "command": "ssh ${config:deckuser}@${config:deckip} -p ${config:deckport} ${config:deckkey} 'python -m ensurepip && python -m pip install --user --upgrade poetry && cd ${config:deckdir}/homebrew/dev/pluginloader/backend && python -m poetry install'",
             "problemMatcher": []
         },
         {
@@ -100,7 +100,7 @@
             "dependsOn": [
                 "checkforsettings"
             ],
-            "command": "ssh deck@${config:deckip} -p ${config:deckport} ${config:deckkey} 'mkdir -p ${config:deckdir}/homebrew/dev/pluginloader && mkdir -p ${config:deckdir}/homebrew/plugins'",
+            "command": "ssh ${config:deckuser}@${config:deckip} -p ${config:deckport} ${config:deckkey} 'mkdir -p ${config:deckdir}/homebrew/dev/pluginloader && mkdir -p ${config:deckdir}/homebrew/plugins'",
             "problemMatcher": []
         },
         {
@@ -108,7 +108,7 @@
             "detail": "Deploy dev PluginLoader to deck",
             "type": "shell",
             "group": "none",
-            "command": "rsync -azp --delete --rsh='ssh -p ${config:deckport} ${config:deckkey}' --exclude='.git/' --exclude='.github/' --exclude='.vscode/' --exclude='frontend/' --exclude='dist/' --exclude='contrib/' --exclude='*.log' --exclude='backend/**/__pycache__/' --exclude='.gitignore' . deck@${config:deckip}:${config:deckdir}/homebrew/dev/pluginloader",
+            "command": "rsync -azp --delete --force --rsh='ssh -p ${config:deckport} ${config:deckkey}' --exclude='.git/' --exclude='.github/' --exclude='.vscode/' --exclude='frontend/' --exclude='dist/' --exclude='contrib/' --exclude='*.log' --exclude='**/__pycache__/' --exclude='.gitignore' . ${config:deckuser}@${config:deckip}:${config:deckdir}/homebrew/dev/pluginloader",
             "problemMatcher": []
         },
         // RUN
@@ -120,7 +120,7 @@
             "dependsOn": [
                 "checkforsettings"
             ],
-            "command": "ssh deck@${config:deckip} -p ${config:deckport} ${config:deckkey} 'export PATH=${config:deckdir}/.local/bin:$PATH; export PLUGIN_PATH=${config:deckdir}/homebrew/plugins; export CHOWN_PLUGIN_PATH=0; export LOG_LEVEL=DEBUG; cd ${config:deckdir}/homebrew/dev/pluginloader/backend; echo '${config:deckpass}' | sudo -SE poetry run sh -c \"cd ${config:deckdir}/homebrew/services; python3 ${config:deckdir}/homebrew/dev/pluginloader/backend/main.py\"'",
+            "command": "ssh ${config:deckuser}@${config:deckip} -p ${config:deckport} ${config:deckkey} 'export PATH=${config:deckdir}/.local/bin:$PATH; export PLUGIN_PATH=${config:deckdir}/homebrew/plugins; export CHOWN_PLUGIN_PATH=0; export LOG_LEVEL=DEBUG; cd ${config:deckdir}/homebrew/dev/pluginloader/backend; echo '${config:deckpass}' | poetry run sh -c \"cd ${config:deckdir}/homebrew/services; sudo -SE env \"PATH=\\$PATH\" python3 ${config:deckdir}/homebrew/dev/pluginloader/backend/main.py\"'",
             "problemMatcher": []
         },
         {

backend/decky_loader/helpers.py

@@ -1,3 +1,4 @@
+from platform import version
 import re
 import ssl
 import uuid
@@ -14,7 +15,7 @@ from aiohttp import ClientSession
 from .localplatform import localplatform
 from .enums import UserType
 from logging import getLogger
-from packaging.version import Version
+from packaging.version import Version # type: ignore
 
 REMOTE_DEBUGGER_UNIT = "steam-web-debug-portforward.service"
 
@@ -36,12 +37,13 @@ def get_csrf_token():
 @middleware
 async def csrf_middleware(request: Request, handler: Handler):
     if str(request.method) == "OPTIONS" or \
-        request.headers.get('Authentication') == csrf_token or \
+        request.headers.get('X-Decky-Auth') == csrf_token or \
         str(request.rel_url) == "/auth/token" or \
         str(request.rel_url).startswith("/plugins/load_main/") or \
         str(request.rel_url).startswith("/static/") or \
         str(request.rel_url).startswith("/steam_resource/") or \
         str(request.rel_url).startswith("/frontend/") or \
+        str(request.rel_url.path) == "/fetch" or \
         str(request.rel_url.path) == "/ws" or \
         assets_regex.match(str(request.rel_url)) or \
         dist_regex.match(str(request.rel_url)) or \
@@ -61,24 +63,27 @@ def mkdir_as_user(path: str):
     localplatform.chown(path)
 
 # Fetches the version of loader
+# TODO THIS IS ABSOLUTELY TERRIBLE AND NEVER SHOULDVE BEEN MERGED! packaging HAS NO TYPES AND WE COULD LITERALLY JUST USE A REGEX!!!!! REWRITE THIS!!!!!!!!!!!!!
 def get_loader_version() -> str:
     try:
         # Normalize Python-style version to conform to Decky style
-        v = Version(importlib.metadata.version("decky_loader"))
+        v = Version(importlib.metadata.version("decky_loader")) # type: ignore
 
-        version_str = f'v{v.major}.{v.minor}.{v.micro}'
+        version_str = f'v{v.major}.{v.minor}.{v.micro}' # type: ignore
 
-        if v.pre:
+        if v.pre: # type: ignore
-            version_str += f'-pre{v.pre[1]}'
+            version_str += f'-pre{v.pre[1]}' # type: ignore
 
-        if v.post:
+        if v.post: # type: ignore
-            version_str += f'-dev{v.post}'
+            version_str += f'-dev{v.post}' # type: ignore
 
         return version_str
     except Exception as e:
         logger.warn(f"Failed to execute get_loader_version(): {str(e)}")
         return "unknown"
 
+user_agent = f"Decky/{get_loader_version()} (https://decky.xyz)"
+
 # returns the appropriate system python paths
 def get_system_pythonpaths() -> list[str]:
     try:

backend/decky_loader/plugin/imports/decky.py

@@ -12,7 +12,7 @@ Some basic migration helpers are available: `migrate_any`, `migrate_settings`, `
 A logging facility `logger` is available which writes to the recommended location.
 """
 
-__version__ = '0.1.0'
+__version__ = '1.0.0'
 
 import os
 import subprocess

backend/decky_loader/plugin/imports/decky.pyi

@@ -12,7 +12,7 @@ Some basic migration helpers are available: `migrate_any`, `migrate_settings`, `
 A logging facility `logger` is available which writes to the recommended location.
 """
 
-__version__ = '0.1.0'
+__version__ = '1.0.0'
 
 import logging
 

backend/decky_loader/plugin/plugin.py

@@ -47,6 +47,7 @@ class PluginWrapper:
 
         self.emitted_event_callback: EmittedEventCallbackType = emit_callback
 
+        # TODO enable this after websocket release
         self.legacy_method_warning = False
 
     def __str__(self) -> str:

backend/decky_loader/utilities.py

@@ -1,6 +1,8 @@
 from __future__ import annotations
 from os import stat_result
 import uuid
+from urllib.parse import unquote
+from json.decoder import JSONDecodeError
 from os.path import splitext
 import re
 from traceback import format_exc
@@ -8,6 +10,7 @@ from stat import FILE_ATTRIBUTE_HIDDEN # type: ignore
 
 from asyncio import StreamReader, StreamWriter, start_server, gather, open_connection
 from aiohttp import ClientSession
+from aiohttp.web import Request, StreamResponse, Response, json_response, post
 from typing import TYPE_CHECKING, Callable, Coroutine, Dict, Any, List, TypedDict
 
 from logging import getLogger
@@ -26,12 +29,17 @@ class FilePickerObj(TypedDict):
     filest: stat_result
     is_dir: bool
 
+decky_header_regex = re.compile("X-Decky-(.*)")
+extra_header_regex = re.compile("X-Decky-Header-(.*)")
+
+excluded_default_headers = ["Host", "Origin", "Sec-Fetch-Site", "Sec-Fetch-Mode", "Sec-Fetch-Dest"]
+
 class Utilities:
     def __init__(self, context: PluginManager) -> None:
         self.context = context
         self.legacy_util_methods: Dict[str, Callable[..., Coroutine[Any, Any, Any]]] = {
             "ping": self.ping,
-            "http_request": self.http_request,
+            "http_request": self.http_request_legacy,
             "install_plugin": self.install_plugin,
             "install_plugins": self.install_plugins,
             "cancel_plugin_install": self.cancel_plugin_install,
@@ -76,9 +84,33 @@ class Utilities:
             context.ws.add_route("utilities/enable_rdt", self.enable_rdt)
             context.ws.add_route("utilities/get_tab_id", self.get_tab_id)
             context.ws.add_route("utilities/get_user_info", self.get_user_info)
-            context.ws.add_route("utilities/http_request", self.http_request)
+            context.ws.add_route("utilities/http_request", self.http_request_legacy)
             context.ws.add_route("utilities/_call_legacy_utility", self._call_legacy_utility)
 
+            context.web_app.add_routes([
+                post("/methods/{method_name}", self._handle_legacy_server_method_call)
+            ])
+
+            for method in ('GET', 'POST', 'PUT', 'DELETE', 'PATCH', 'HEAD'):
+                context.web_app.router.add_route(method, "/fetch", self.http_request)
+
+
+    async def _handle_legacy_server_method_call(self, request: Request) -> Response:
+        method_name = request.match_info["method_name"]
+        try:
+            args = await request.json()
+        except JSONDecodeError:
+            args = {}
+        res = {}
+        try:
+            r = await self.legacy_util_methods[method_name](**args)
+            res["result"] = r
+            res["success"] = True
+        except Exception as e:
+            res["result"] = str(e)
+            res["success"] = False
+        return json_response(res)
+
     async def _call_legacy_utility(self, method_name: str, kwargs: Dict[Any, Any]) -> Any:
         self.logger.debug(f"Calling utility {method_name} with legacy kwargs");
         res: Dict[Any, Any] = {}
@@ -114,7 +146,63 @@ class Utilities:
     async def uninstall_plugin(self, name: str):
         return await self.context.plugin_browser.uninstall_plugin(name)
 
-    async def http_request(self, method: str, url: str, extra_opts: Any = {}):
+    # Loosely based on https://gist.github.com/mosquito/4dbfacd51e751827cda7ec9761273e95#file-proxy-py
+    async def http_request(self, req: Request) -> StreamResponse:
+        if req.headers.get('X-Decky-Auth', '') != helpers.get_csrf_token() and req.query.get('auth', '') != helpers.get_csrf_token():
+            return Response(text='Forbidden', status=403)
+
+        url = req.headers["X-Decky-Fetch-URL"] if "X-Decky-Fetch-URL" in req.headers else unquote(req.query.get('fetch_url', ''))
+        self.logger.info(f"Preparing {req.method} request to {url}")
+
+        headers = dict(req.headers)
+
+        headers["User-Agent"] = helpers.user_agent
+
+        for excluded_header in excluded_default_headers:
+            self.logger.debug(f"Excluding default header {excluded_header}")
+            if excluded_header in headers:
+                del headers[excluded_header]
+
+        if "X-Decky-Fetch-Excluded-Headers" in req.headers:
+            for excluded_header in req.headers["X-Decky-Fetch-Excluded-Headers"].split(", "):
+                self.logger.debug(f"Excluding header {excluded_header}")
+                if excluded_header in headers:
+                    del headers[excluded_header]
+
+        for header in req.headers:
+            match = extra_header_regex.search(header)
+            if match:
+                header_name = match.group(1)
+                header_value = req.headers[header]
+                self.logger.debug(f"Adding extra header {header_name}: {header_value}")
+                headers[header_name] = header_value
+
+        for header in list(headers.keys()):
+            match = decky_header_regex.search(header)
+            if match:
+                self.logger.debug(f"Removing decky header {header} from request")
+                del headers[header]
+
+        self.logger.debug(f"Final request headers: {headers}")
+
+        body = await req.read() # TODO can this also be streamed?
+
+        async with ClientSession() as web:
+            async with web.request(req.method, url, headers=headers, data=body, ssl=helpers.get_ssl_context()) as web_res:
+                res = StreamResponse(headers=web_res.headers, status=web_res.status)
+                if web_res.headers.get('Transfer-Encoding', '').lower() == 'chunked':
+                    res.enable_chunked_encoding()
+
+                await res.prepare(req)
+                self.logger.debug(f"Starting stream for {url}")
+                async for data in web_res.content.iter_any():
+                    await res.write(data)
+                    if data:
+                        await res.drain()
+                self.logger.debug(f"Finished stream for {url}")
+        return res
+
+    async def http_request_legacy(self, method: str, url: str, extra_opts: Any = {}):
         async with ClientSession() as web:
             res = await web.request(method, url, ssl=helpers.get_ssl_context(), **extra_opts)
             text = await res.text()

backend/decky_loader/wsrouter.py

@@ -117,7 +117,7 @@ class WSRouter:
                                     create_task(self._call_route(data["route"], data["args"], data["id"]))
                                 else:
                                     error = {"error":f'Route {data["route"]} does not exist.', "name": "RouteNotFoundError", "traceback": None}
-                                    create_task(self.write({"type": MessageType.ERROR.value, "id": data["id"], "message": error}))
+                                    create_task(self.write({"type": MessageType.ERROR.value, "id": data["id"], "error": error}))
                             case _:
                                 self.logger.error("Unknown message type", data)
         finally:

backend/pyinstaller.spec

@@ -9,7 +9,7 @@ a = Analysis(
         ('locales', 'locales'),
         ('static', 'static'),
     ] + copy_metadata('decky_loader'),
-    hiddenimports=['logging.handlers', 'sqlite3', 'decky_plugin'],
+    hiddenimports=['logging.handlers', 'sqlite3', 'decky_plugin' 'decky'],
 )
 pyz = PYZ(a.pure, a.zipped_data)
 

backend/pyproject.toml

@@ -13,12 +13,13 @@ include = ["decky_loader/static/*"]
 [tool.poetry.dependencies]
 python = ">=3.10,<3.13"
 
-aiohttp = "^3.8.5"
+aiohttp = "^3.9.5"
 aiohttp-jinja2 = "^1.5.1"
 aiohttp-cors = "^0.7.0"
 watchdog = "^2.1.7"
 certifi = "*"
 packaging = "^23.2"
+multidict = "^6.0.5"
 
 [tool.poetry.group.dev.dependencies]
 pyinstaller = "^5.13.0"

frontend/src/plugin-loader.tsx

@@ -42,7 +42,7 @@ const FilePicker = lazy(() => import('./components/modals/filepicker'));
 
 declare global {
   interface Window {
-    __DECKY_SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED_deckyPluginBackendAPIInit?: {
+    __DECKY_SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED_deckyLoaderAPIInit?: {
       connect: (version: number, key: string) => any; // Returns the backend API used above, no real point adding types to this.
     };
   }
@@ -51,6 +51,10 @@ declare global {
 /** Map of event names to event listeners */
 type listenerMap = Map<string, Set<(...args: any) => any>>;
 
+interface DeckyRequestInit extends RequestInit {
+  excludedHeaders: string[];
+}
+
 const callPluginMethod = DeckyBackend.callable<[pluginName: string, method: string, ...args: any], any>(
   'loader/call_plugin_method',
 );
@@ -357,7 +361,7 @@ class PluginLoader extends Logger {
           let res = await fetch(`http://127.0.0.1:1337/plugins/${name}/frontend_bundle`, {
             credentials: 'include',
             headers: {
-              Authentication: deckyAuthToken,
+              'X-Decky-Auth': deckyAuthToken,
             },
           });
           if (res.ok) {
@@ -484,12 +488,31 @@ class PluginLoader extends Logger {
     });
   }
 
-  /* TODO replace with the following flow (or similar) so we can reuse the JS Fetch API
+  // Useful for audio/video streams
-        frontend --request URL only--> backend (ws method)
+  getExternalResourceURL(url: string) {
-        backend --new temporary backend URL--> frontend (ws response)
+    return `http://127.0.0.1:1337/fetch?auth=${deckyAuthToken}&fetch_url=${encodeURIComponent(url)}`;
-        frontend <--> backend <--> target URL (over http!)
+  }
-      */
+
-  async fetchNoCors(url: string, request: any = {}) {
+  // Same syntax as fetch but only supports the url-based syntax and an object for headers since it's the most common usage pattern
+  fetch(input: string, init?: DeckyRequestInit | undefined): Promise<Response> {
+    const headers: { [name: string]: string } = {
+      ...(init?.headers as { [name: string]: string }),
+      'X-Decky-Auth': deckyAuthToken,
+      'X-Decky-Fetch-URL': input,
+    };
+
+    if (init?.excludedHeaders) {
+      headers['X-Decky-Fetch-Excluded-Headers'] = init.excludedHeaders.join(', ');
+    }
+
+    return fetch('http://127.0.0.1:1337/fetch', {
+      ...init,
+      credentials: 'include',
+      headers,
+    });
+  }
+
+  async legacyFetchNoCors(url: string, request: any = {}) {
     let method: string;
     const req = { headers: {}, ...request, data: request.body };
     req?.body && delete req.body;
@@ -513,10 +536,10 @@ class PluginLoader extends Logger {
 
   initPluginBackendAPI() {
     // Things will break *very* badly if plugin code touches this outside of @decky/backend, so lets make that clear.
-    window.__DECKY_SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED_deckyPluginBackendAPIInit = {
+    window.__DECKY_SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED_deckyLoaderAPIInit = {
       connect: (version: number, pluginName: string) => {
-        if (version <= 0) {
+        if (version < 1 || version > 1) {
-          throw new Error(`Plugin ${pluginName} requested invalid backend api version ${version}.`);
+          throw new Error(`Plugin ${pluginName} requested unsupported backend api version ${version}.`);
         }
 
         const eventListeners: listenerMap = new Map();
@@ -543,9 +566,22 @@ class PluginLoader extends Logger {
               set?.delete(listener);
             }
           },
+          openFilePicker: this.openFilePicker.bind(this),
+          executeInTab: DeckyBackend.callable<
+            [tab: String, runAsync: Boolean, code: string],
+            { success: boolean; result: any }
+          >('utilities/execute_in_tab'),
+          fetch: this.fetch.bind(this),
+          getExternalResourceURL: this.getExternalResourceURL.bind(this),
+          injectCssIntoTab: DeckyBackend.callable<[tab: string, style: string], string>(
+            'utilities/inject_css_into_tab',
+          ),
+          removeCssFromTab: DeckyBackend.callable<[tab: string, cssId: string]>('utilities/remove_css_from_tab'),
+          routerHook: this.routerHook,
+          toaster: this.toaster,
         };
 
-        this.debug(`${pluginName} connected to backend API.`);
+        this.debug(`${pluginName} connected to loader API.`);
         return backendAPI;
       },
     };
@@ -591,7 +627,7 @@ class PluginLoader extends Logger {
           args,
         );
       },
-      fetchNoCors: this.fetchNoCors,
+      fetchNoCors: this.legacyFetchNoCors,
       executeInTab: DeckyBackend.callable<
         [tab: String, runAsync: Boolean, code: string],
         { success: boolean; result: any }

frontend/src/start.tsx

@@ -32,7 +32,7 @@ declare global {
       backend: {
         loadPath: 'http://127.0.0.1:1337/locales/{{lng}}.json',
         customHeaders: {
-          Authentication: deckyAuthToken,
+          'X-Decky-Auth': deckyAuthToken,
         },
         requestOptions: {
           credentials: 'include',

frontend/src/wsrouter.ts

@@ -30,7 +30,7 @@ interface ReplyMessage {
 
 interface ErrorMessage {
   type: MessageType.ERROR;
-  error: { name: string; message: string; traceback: string | null };
+  error: { name: string; error: string; traceback: string | null };
   id: number;
 }
 
@@ -40,8 +40,8 @@ interface ErrorMessage {
 export class PyError extends Error {
   pythonTraceback: string | null;
 
-  constructor(name: string, message: string, traceback: string | null) {
+  constructor(name: string, error: string, traceback: string | null) {
-    super(message);
+    super(error);
     this.name = `Python ${name}`;
     if (traceback) {
       // traceback will always start with `Traceback (most recent call last):`
@@ -142,7 +142,7 @@ export class WSRouter extends Logger {
 
         case MessageType.ERROR:
           if (this.runningCalls.has(data.id)) {
-            let err = new PyError(data.error.name, data.error.message, data.error.traceback);
+            let err = new PyError(data.error.name, data.error.error, data.error.traceback);
             this.runningCalls.get(data.id)!.reject(err);
             this.runningCalls.delete(data.id);
             this.debug(`Rejected PY call ${data.id} with error`, data.error);