diff --git a/.gitignore b/.gitignore index 7855d3b36e..ab697c25d1 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,7 @@ dist/ # this is for jetbrain IDEs .idea/ /puter + +# Local Netlify folder +.netlify +src/emulator/release/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 40a50189cf..7b01a93997 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,15 +45,18 @@ If you'd like to contribute code to Puter, you need to fork the project and subm We'll review your pull request and work with you to get your changes merged into the project. +## Repository Structure + +![file structure](./doc/File%20Structure.drawio.png) + ## Your first code contribution We maintain a list of issues that are good for first-time contributors. You can find these issues by searching for the [`good first issue`](https://github.com/HeyPuter/puter/issues?q=is%3Aissue+is%3Aopen+label%3A%22good+first+issue%22) label in our [GitHub repository](https://github.com/HeyPuter/puter). These issues are designed to be relatively easy to fix, and we're happy to help you get started. Pick an issue that interests you, and leave a comment on the issue to let us know you're working on it. -
- ## Documentation for Contributors -See [doc/contributors/index.md](./doc/contributors/index.md) for more information. +### Backend +See [src/backend/CONTRIBUTING.md](src/backend/CONTRIBUTING.md)
diff --git a/awesome/#DoesItRunPuter.md b/awesome/#DoesItRunPuter.md index bb737e5717..6c6cc7e9e7 100644 --- a/awesome/#DoesItRunPuter.md +++ b/awesome/#DoesItRunPuter.md @@ -13,3 +13,4 @@ - [Steam Deck](https://twitter.com/everythingSung/status/1782162352403828793) - [Ladybird Browser](https://x.com/HeyPuter/status/1810783504503800035) - [Garry's Mod](https://x.com/HeyPuter/status/1850587712786722862) +- [Samsung Q88BA](https://x.com/AmirIsAround/status/1862614583263076540) diff --git a/doc/File Structure.drawio b/doc/File Structure.drawio new file mode 100644 index 0000000000..2af3e4a1c6 --- /dev/null +++ b/doc/File Structure.drawio @@ -0,0 +1,214 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/doc/File Structure.drawio.png b/doc/File Structure.drawio.png new file mode 100644 index 0000000000..16de66fa4f Binary files /dev/null and b/doc/File Structure.drawio.png differ diff --git a/doc/contributors/extensions.md b/doc/contributors/extensions.md index 12cd44d636..e95570e644 100644 --- a/doc/contributors/extensions.md +++ b/doc/contributors/extensions.md @@ -1,2 +1,3 @@ -### `vscode` -- `es6-string-html` +## Puter Extensions + +See the [Wiki Page](https://github.com/HeyPuter/puter/wiki/ex_extensions) diff --git a/doc/contributors/vscode.md b/doc/contributors/vscode.md new file mode 100644 index 0000000000..12cd44d636 --- /dev/null +++ b/doc/contributors/vscode.md @@ -0,0 +1,2 @@ +### `vscode` +- `es6-string-html` diff --git a/doc/i18n/README.zh.md b/doc/i18n/README.zh.md index bf04949934..7022380f92 100644 --- a/doc/i18n/README.zh.md +++ b/doc/i18n/README.zh.md @@ -89,6 +89,15 @@ docker compose up ```
+## 宝塔面板Docker一键部署(推荐) + +1. 安装宝塔面板9.2.0及以上版本,前往 [宝塔面板](https://www.bt.cn/new/download.html?r=dk_puter) 官网,选择正式版的脚本下载安装 + +2. 安装后登录宝塔面板,在左侧菜单栏中点击 `Docker`,首次进入会提示安装`Docker`服务,点击立即安装,按提示完成安装 + +3. 安装完成后在应用商店中搜索`puter`,点击安装,配置域名等基本信息即可完成安装 + + ### ☁️ Puter.com Puter 可以作为托管服务使用,访问 [**puter.com**](https://puter.com)。 diff --git a/experiments/x86emu/www/main.js b/experiments/x86emu/www/main.js index 69df0af813..b936876ed6 100755 --- a/experiments/x86emu/www/main.js +++ b/experiments/x86emu/www/main.js @@ -17,7 +17,6 @@ * along with this program. If not, see . */ -#!/usr/bin/env node /* * Copyright (C) 2024 Puter Technologies Inc. * diff --git a/mods/mods_available/kdmod/ShareTestService.js b/mods/mods_available/kdmod/ShareTestService.js index 3e8e61af92..94bf87bea9 100644 --- a/mods/mods_available/kdmod/ShareTestService.js +++ b/mods/mods_available/kdmod/ShareTestService.js @@ -21,7 +21,6 @@ // we have these things registered in "useapi". const { get_user, - generate_system_fsentries, invalidate_cached_user, deleteUser, } = require('../../../src/backend/src/helpers.js'); @@ -146,7 +145,8 @@ class ShareTestService extends use.Service { ], ); const user = await get_user({ username }); - await generate_system_fsentries(user); + const svc_user = this.services.get('user'); + await svc_user.generate_default_fsentries({ user }); invalidate_cached_user(user); return user; } diff --git a/package-lock.json b/package-lock.json index 93984e84a2..7be249a69c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17,6 +17,7 @@ "javascript-time-ago": "^2.5.11", "json-colorizer": "^3.0.1", "open": "^10.1.0", + "sharp": "^0.33.5", "simple-git": "^3.25.0", "string-template": "^1.0.0", "uuid": "^9.0.1" @@ -1897,13 +1898,12 @@ } }, "node_modules/@babel/code-frame": { - "version": "7.24.7", - "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.24.7.tgz", - "integrity": "sha512-BcYH1CVJBO9tvyIZ2jVeXgSIMvGZ2FDRvDdOIVQyuklNKSsx+eppDEBq/g47Ayw+RqNFE+URvOShmf+f/qwAlA==", - "dev": true, - "license": "MIT", + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.26.2.tgz", + "integrity": "sha512-RJlIHRueQgwWitWgF8OdFYGZX328Ax5BCemNGlqHfplnRT9ESi8JkFlvaVYbS+UubVY6dpv87Fs2u5M29iNFVQ==", "dependencies": { - "@babel/highlight": "^7.24.7", + "@babel/helper-validator-identifier": "^7.25.9", + "js-tokens": "^4.0.0", "picocolors": "^1.0.0" }, "engines": { @@ -1969,15 +1969,15 @@ } }, "node_modules/@babel/generator": { - "version": "7.24.7", - "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.24.7.tgz", - "integrity": "sha512-oipXieGC3i45Y1A41t4tAqpnEZWgB/lC6Ehh6+rOviR5XWpTtMmLN+fGjz9vOiNRt0p6RtO6DtD0pdU3vpqdSA==", - "license": "MIT", + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.26.2.tgz", + "integrity": "sha512-zevQbhbau95nkoxSq3f/DC/SC+EEOUZd3DYqfSkMhY2/wfSeaHV1Ew4vk8e+x8lja31IbyuUa2uQ3JONqKbysw==", "dependencies": { - "@babel/types": "^7.24.7", + "@babel/parser": "^7.26.2", + "@babel/types": "^7.26.0", "@jridgewell/gen-mapping": "^0.3.5", "@jridgewell/trace-mapping": "^0.3.25", - "jsesc": "^2.5.1" + "jsesc": "^3.0.2" }, "engines": { "node": ">=6.9.0" @@ -2023,33 +2023,6 @@ "node": ">=6.9.0" } }, - "node_modules/@babel/helper-function-name": { - "version": "7.24.7", - "resolved": "https://registry.npmjs.org/@babel/helper-function-name/-/helper-function-name-7.24.7.tgz", - "integrity": "sha512-FyoJTsj/PEUWu1/TYRiXTIHc8lbw+TDYkZuoE43opPS5TrI7MyONBE1oNvfguEXAD9yhQRrVBnXdXzSLQl9XnA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/template": "^7.24.7", - "@babel/types": "^7.24.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/helper-hoist-variables": { - "version": "7.24.7", - "resolved": "https://registry.npmjs.org/@babel/helper-hoist-variables/-/helper-hoist-variables-7.24.7.tgz", - "integrity": "sha512-MJJwhkoGy5c4ehfoRyrJ/owKeMl19U54h27YYftT0o2teQ3FJ3nQUf/I3LlJsX4l3qlw7WRXUmiyajvHXoTubQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/types": "^7.24.7" - }, - "engines": { - "node": ">=6.9.0" - } - }, "node_modules/@babel/helper-module-imports": { "version": "7.24.7", "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.24.7.tgz", @@ -2112,19 +2085,17 @@ } }, "node_modules/@babel/helper-string-parser": { - "version": "7.24.7", - "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.24.7.tgz", - "integrity": "sha512-7MbVt6xrwFQbunH2DNQsAP5sTGxfqQtErvBIvIMi6EQnbgUOuVYanvREcmFrOPhoXBrTtjhhP+lW+o5UfK+tDg==", - "license": "MIT", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.25.9.tgz", + "integrity": "sha512-4A/SCr/2KLd5jrtOMFzaKjVtAei3+2r/NChoBNoZ3EyP/+GlhoaEGoWOZUmFmoITP7zOJyHIMm+DYRd8o3PvHA==", "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/helper-validator-identifier": { - "version": "7.24.7", - "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.24.7.tgz", - "integrity": "sha512-rR+PBcQ1SMQDDyF6X0wxtG8QyLCgUB0eRAGguqRLfkCA87l7yAP7ehq8SNj96OOGTO8OBV70KhuFYcIkHXOg0w==", - "license": "MIT", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.25.9.tgz", + "integrity": "sha512-Ed61U6XJc3CVRfkERJWDz4dJwKe7iLmmJsbOGu9wSloNSFttHV0I8g6UAgb7qnK5ly5bGLPd4oXZlxCdANBOWQ==", "engines": { "node": ">=6.9.0" } @@ -2153,82 +2124,13 @@ "node": ">=6.9.0" } }, - "node_modules/@babel/highlight": { - "version": "7.24.7", - "resolved": "https://registry.npmjs.org/@babel/highlight/-/highlight-7.24.7.tgz", - "integrity": "sha512-EStJpq4OuY8xYfhGVXngigBJRWxftKX9ksiGDnmlY3o7B/V7KIAc9X4oiK87uPJSc/vs5L869bem5fhZa8caZw==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/helper-validator-identifier": "^7.24.7", - "chalk": "^2.4.2", - "js-tokens": "^4.0.0", - "picocolors": "^1.0.0" - }, - "engines": { - "node": ">=6.9.0" - } - }, - "node_modules/@babel/highlight/node_modules/ansi-styles": { - "version": "3.2.1", - "resolved": "https://registry.npmjs.org/ansi-styles/-/ansi-styles-3.2.1.tgz", - "integrity": "sha512-VT0ZI6kZRdTh8YyJw3SMbYm/u+NqfsAxEpWO0Pf9sq8/e94WxxOpPKx9FR1FlyCtOVDNOQ+8ntlqFxiRc+r5qA==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-convert": "^1.9.0" - }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/chalk": { - "version": "2.4.2", - "resolved": "https://registry.npmjs.org/chalk/-/chalk-2.4.2.tgz", - "integrity": "sha512-Mti+f9lpJNcwF4tWV8/OrTTtF1gZi+f8FqlyAdouralcFWFQWF2+NgCHShjkCb+IFBLq9buZwE1xckQU4peSuQ==", - "dev": true, - "license": "MIT", + "node_modules/@babel/parser": { + "version": "7.26.2", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.26.2.tgz", + "integrity": "sha512-DWMCZH9WA4Maitz2q21SRKHo9QXZxkDsbNZoVD62gusNtNBBqDg9i7uOhASfTfIGNzW+O+r7+jAlM8dwphcJKQ==", "dependencies": { - "ansi-styles": "^3.2.1", - "escape-string-regexp": "^1.0.5", - "supports-color": "^5.3.0" + "@babel/types": "^7.26.0" }, - "engines": { - "node": ">=4" - } - }, - "node_modules/@babel/highlight/node_modules/color-convert": { - "version": "1.9.3", - "resolved": "https://registry.npmjs.org/color-convert/-/color-convert-1.9.3.tgz", - "integrity": "sha512-QfAUtd+vFdAtFQcC8CCyYt1fYWxSqAiK2cSD6zDB8N3cpsEBAvRxp9zOGg6G/SHHJYAT88/az/IuDGALsNVbGg==", - "dev": true, - "license": "MIT", - "dependencies": { - "color-name": "1.1.3" - } - }, - "node_modules/@babel/highlight/node_modules/color-name": { - "version": "1.1.3", - "resolved": "https://registry.npmjs.org/color-name/-/color-name-1.1.3.tgz", - "integrity": "sha512-72fSenhMw2HZMTVHeCA9KCmpEIbzWiQsjN+BHcBbS9vr1mtt+vJjPdksIBNUmKAW8TFUDPJK5SUU3QhE9NEXDw==", - "dev": true, - "license": "MIT" - }, - "node_modules/@babel/highlight/node_modules/escape-string-regexp": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", - "integrity": "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.0" - } - }, - "node_modules/@babel/parser": { - "version": "7.24.7", - "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.24.7.tgz", - "integrity": "sha512-9uUYRm6OqQrCqQdG1iCBwBPZgN8ciDBro2nIOFaiRz1/BCxaI7CNvQbDHvsArAC7Tw9Hda/B3U+6ui9u4HWXPw==", - "license": "MIT", "bin": { "parser": "bin/babel-parser.js" }, @@ -2250,35 +2152,28 @@ } }, "node_modules/@babel/template": { - "version": "7.24.7", - "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.24.7.tgz", - "integrity": "sha512-jYqfPrU9JTF0PmPy1tLYHW4Mp4KlgxJD9l2nP9fD6yT/ICi554DmrWBAEYpIelzjHf1msDP3PxJIRt/nFNfBig==", - "dev": true, - "license": "MIT", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.25.9.tgz", + "integrity": "sha512-9DGttpmPvIxBb/2uwpVo3dqJ+O6RooAFOS+lB+xDqoE2PVCE8nfoHMdZLpfCQRLwvohzXISPZcgxt80xLfsuwg==", "dependencies": { - "@babel/code-frame": "^7.24.7", - "@babel/parser": "^7.24.7", - "@babel/types": "^7.24.7" + "@babel/code-frame": "^7.25.9", + "@babel/parser": "^7.25.9", + "@babel/types": "^7.25.9" }, "engines": { "node": ">=6.9.0" } }, "node_modules/@babel/traverse": { - "version": "7.24.7", - "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.24.7.tgz", - "integrity": "sha512-yb65Ed5S/QAcewNPh0nZczy9JdYXkkAbIsEo+P7BE7yO3txAY30Y/oPa3QkQ5It3xVG2kpKMg9MsdxZaO31uKA==", - "dev": true, - "license": "MIT", - "dependencies": { - "@babel/code-frame": "^7.24.7", - "@babel/generator": "^7.24.7", - "@babel/helper-environment-visitor": "^7.24.7", - "@babel/helper-function-name": "^7.24.7", - "@babel/helper-hoist-variables": "^7.24.7", - "@babel/helper-split-export-declaration": "^7.24.7", - "@babel/parser": "^7.24.7", - "@babel/types": "^7.24.7", + "version": "7.25.9", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.25.9.tgz", + "integrity": "sha512-ZCuvfwOwlz/bawvAuvcj8rrithP2/N55Tzz342AkTvq4qaWbGfmCk/tKhNaV2cthijKrPAA8SRJV5WWe7IBMJw==", + "dependencies": { + "@babel/code-frame": "^7.25.9", + "@babel/generator": "^7.25.9", + "@babel/parser": "^7.25.9", + "@babel/template": "^7.25.9", + "@babel/types": "^7.25.9", "debug": "^4.3.1", "globals": "^11.1.0" }, @@ -2290,21 +2185,18 @@ "version": "11.12.0", "resolved": "https://registry.npmjs.org/globals/-/globals-11.12.0.tgz", "integrity": "sha512-WOBp/EEGUiIsJSp7wcv/y6MO+lV9UoncWqxuFfm8eBwzWNgyfBd6Gz+IeKQ9jCmyhoH99g15M3T+QaVHFjizVA==", - "dev": true, "license": "MIT", "engines": { "node": ">=4" } }, "node_modules/@babel/types": { - "version": "7.24.7", - "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.24.7.tgz", - "integrity": "sha512-XEFXSlxiG5td2EJRe8vOmRbaXVgfcBlszKujvVmWIK/UpywWljQCfzAv3RQCGujWQ1RD4YYWEAqDXfuJiy8f5Q==", - "license": "MIT", + "version": "7.26.0", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.26.0.tgz", + "integrity": "sha512-Z/yiTPj+lDVnF7lWeKCIJzaIkI0vYO87dMpZ4bg4TDrFe4XXLFWL1TbXU27gBP3QccxV9mZICCrnjnYlJjXHOA==", "dependencies": { - "@babel/helper-string-parser": "^7.24.7", - "@babel/helper-validator-identifier": "^7.24.7", - "to-fast-properties": "^2.0.0" + "@babel/helper-string-parser": "^7.25.9", + "@babel/helper-validator-identifier": "^7.25.9" }, "engines": { "node": ">=6.9.0" @@ -2339,6 +2231,15 @@ "node": ">=10.0.0" } }, + "node_modules/@emnapi/runtime": { + "version": "1.3.1", + "resolved": "https://registry.npmjs.org/@emnapi/runtime/-/runtime-1.3.1.tgz", + "integrity": "sha512-kEBmG8KyqtxJZv+ygbEim+KCGtIq1fC22Ms3S4ziXmYKm8uyoLX0MHONVKwp+9opg390VaKRNt4a7A9NwmpNhw==", + "optional": true, + "dependencies": { + "tslib": "^2.4.0" + } + }, "node_modules/@eslint-community/eslint-utils": { "version": "4.4.0", "resolved": "https://registry.npmjs.org/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz", @@ -2596,8 +2497,8 @@ "form-data": "^4.0.0" } }, - "node_modules/@heyputer/parsely": { - "resolved": "src/parsely", + "node_modules/@heyputer/parsers": { + "resolved": "src/parsers", "link": true }, "node_modules/@heyputer/phoenix": { @@ -2666,6 +2567,348 @@ "url": "https://github.com/sponsors/nzakas" } }, + "node_modules/@img/sharp-darwin-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-arm64/-/sharp-darwin-arm64-0.33.5.tgz", + "integrity": "sha512-UT4p+iz/2H4twwAoLCqfA9UH5pI6DggwKEGuaPy7nCVQ8ZsiY5PIcrRvD1DzuY3qYL07NtIQcWnBSY/heikIFQ==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-darwin-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-darwin-x64/-/sharp-darwin-x64-0.33.5.tgz", + "integrity": "sha512-fyHac4jIc1ANYGRDxtiqelIbdWkIuQaI84Mv45KvGRRxSAa7o7d1ZKAOBaYbnepLC1WqxfpimdeWfvqqSGwR2Q==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-darwin-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-libvips-darwin-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-arm64/-/sharp-libvips-darwin-arm64-1.0.4.tgz", + "integrity": "sha512-XblONe153h0O2zuFfTAbQYAX2JhYmDHeWikp1LM9Hul9gVPjFY427k6dFEcOL72O01QxQsWi761svJ/ev9xEDg==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-darwin-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-darwin-x64/-/sharp-libvips-darwin-x64-1.0.4.tgz", + "integrity": "sha512-xnGR8YuZYfJGmWPvmlunFaWJsb9T/AO2ykoP3Fz/0X5XV2aoYBPkX6xqCQvUTKKiLddarLaxpzNe+b1hjeWHAQ==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "darwin" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm/-/sharp-libvips-linux-arm-1.0.5.tgz", + "integrity": "sha512-gvcC4ACAOPRNATg/ov8/MnbxFDJqf/pDePbBnuBDcjsI8PssmjoKMAz4LtLaVi+OnSb5FK/yIOamqDwGmXW32g==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-arm64/-/sharp-libvips-linux-arm64-1.0.4.tgz", + "integrity": "sha512-9B+taZ8DlyyqzZQnoeIvDVR/2F4EbMepXMc/NdVbkzsJbzkUjhXv/70GQJ7tdLA4YJgNP25zukcxpX2/SueNrA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-s390x": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-s390x/-/sharp-libvips-linux-s390x-1.0.4.tgz", + "integrity": "sha512-u7Wz6ntiSSgGSGcjZ55im6uvTrOxSIS8/dgoVMoiGE9I6JAfU50yH5BoDlYA1tcuGS7g/QNtetJnxA6QEsCVTA==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linux-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linux-x64/-/sharp-libvips-linux-x64-1.0.4.tgz", + "integrity": "sha512-MmWmQ3iPFZr0Iev+BAgVMb3ZyC4KeFc3jFxnNbEPas60e1cIfevbtuyf9nDGIzOaW9PdnDciJm+wFFaTlj5xYw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-arm64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-arm64/-/sharp-libvips-linuxmusl-arm64-1.0.4.tgz", + "integrity": "sha512-9Ti+BbTYDcsbp4wfYib8Ctm1ilkugkA/uscUn6UXK1ldpC1JjiXbLfFZtRlBhjPZ5o1NCLiDbg8fhUPKStHoTA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-libvips-linuxmusl-x64": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/@img/sharp-libvips-linuxmusl-x64/-/sharp-libvips-linuxmusl-x64-1.0.4.tgz", + "integrity": "sha512-viYN1KX9m+/hGkJtvYYp+CCLgnJXwiQB39damAO7WMdKWlIhmYTfHjwSbQeUK/20vY154mwezd9HflVFM1wVSw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-linux-arm": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm/-/sharp-linux-arm-0.33.5.tgz", + "integrity": "sha512-JTS1eldqZbJxjvKaAkxhZmBqPRGmxgu+qFKSInv8moZ2AmT5Yib3EQ1c6gp493HvrvV8QgdOXdyaIBrhvFhBMQ==", + "cpu": [ + "arm" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm": "1.0.5" + } + }, + "node_modules/@img/sharp-linux-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-arm64/-/sharp-linux-arm64-0.33.5.tgz", + "integrity": "sha512-JMVv+AMRyGOHtO1RFBiJy/MBsgz0x4AWrT6QoEVVTyh1E39TrCUpTRI7mx9VksGX4awWASxqCYLCV4wBZHAYxA==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-s390x": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-s390x/-/sharp-linux-s390x-0.33.5.tgz", + "integrity": "sha512-y/5PCd+mP4CA/sPDKl2961b+C9d+vPAveS33s6Z3zfASk2j5upL6fXVPZi7ztePZ5CuH+1kW8JtvxgbuXHRa4Q==", + "cpu": [ + "s390x" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-s390x": "1.0.4" + } + }, + "node_modules/@img/sharp-linux-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linux-x64/-/sharp-linux-x64-0.33.5.tgz", + "integrity": "sha512-opC+Ok5pRNAzuvq1AG0ar+1owsu842/Ab+4qvU879ippJBHvyY5n2mxF1izXqkPYlGuP/M556uh53jRLJmzTWA==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linux-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-arm64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-arm64/-/sharp-linuxmusl-arm64-0.33.5.tgz", + "integrity": "sha512-XrHMZwGQGvJg2V/oRSUfSAfjfPxO+4DkiRh6p2AFjLQztWUuY/o8Mq0eMQVIY7HJ1CDQUJlxGGZRw1a5bqmd1g==", + "cpu": [ + "arm64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4" + } + }, + "node_modules/@img/sharp-linuxmusl-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-linuxmusl-x64/-/sharp-linuxmusl-x64-0.33.5.tgz", + "integrity": "sha512-WT+d/cgqKkkKySYmqoZ8y3pxx7lx9vVejxW/W4DOFMYVSkErR+w7mf2u8m/y4+xHe7yY9DAXQMWQhpnMuFfScw==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-libvips-linuxmusl-x64": "1.0.4" + } + }, + "node_modules/@img/sharp-wasm32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-wasm32/-/sharp-wasm32-0.33.5.tgz", + "integrity": "sha512-ykUW4LVGaMcU9lu9thv85CbRMAwfeadCJHRsg2GmeRa/cJxsVY9Rbd57JcMxBkKHag5U/x7TSBpScF4U8ElVzg==", + "cpu": [ + "wasm32" + ], + "optional": true, + "dependencies": { + "@emnapi/runtime": "^1.2.0" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-ia32": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-ia32/-/sharp-win32-ia32-0.33.5.tgz", + "integrity": "sha512-T36PblLaTwuVJ/zw/LaH0PdZkRz5rd3SmMHX8GSmR7vtNSP5Z6bQkExdSK7xGWyxLw4sUknBuugTelgw2faBbQ==", + "cpu": [ + "ia32" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, + "node_modules/@img/sharp-win32-x64": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/@img/sharp-win32-x64/-/sharp-win32-x64-0.33.5.tgz", + "integrity": "sha512-MpY/o8/8kj+EcnxwvrP4aTJSWw/aZ7JIGR4aBeZkZw5B7/Jn+tY9/VNwtcoGmdT7GfggGIU4kygOMSbYnOrAbg==", + "cpu": [ + "x64" + ], + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + } + }, "node_modules/@istanbuljs/load-nyc-config": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/@istanbuljs/load-nyc-config/-/load-nyc-config-1.1.0.tgz", @@ -7800,9 +8043,9 @@ } }, "node_modules/axios": { - "version": "1.7.7", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.7.tgz", - "integrity": "sha512-S4kL7XrjgBmvdGut0sN3yJxqYzrDOnivkBiN0OFs6hLiUam3UPvswUo0kqGyhqUZGEOytHyumEdXsAkgCOUf3Q==", + "version": "1.7.8", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.7.8.tgz", + "integrity": "sha512-Uu0wb7KNqK2t5K+YQyVCLM76prD5sRFjKHbJYCP1J7JFGEQ6nN7HWn9+04LAeiJ3ji54lgS/gZCH1oxyrf1SPw==", "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.0", @@ -8634,6 +8877,10 @@ "resolved": "tools/comment-parser", "link": true }, + "node_modules/comment-writer": { + "resolved": "tools/comment-writer", + "link": true + }, "node_modules/commondir": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/commondir/-/commondir-1.0.1.tgz", @@ -8843,10 +9090,6 @@ "node": ">= 0.6" } }, - "node_modules/contextlink": { - "resolved": "src/contextlink", - "link": true - }, "node_modules/convert-source-map": { "version": "1.9.0", "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-1.9.0.tgz", @@ -9215,7 +9458,6 @@ "version": "1.5.3", "resolved": "https://registry.npmjs.org/dedent/-/dedent-1.5.3.tgz", "integrity": "sha512-NHQtfOOW68WD8lgypbLA5oT+Bt0xXJhiYvoR6SmmNXZfpzOGXwdKWmcwG8N7PwVVWV3eF/68nmD9BaJSsTBhyQ==", - "license": "MIT", "peerDependencies": { "babel-plugin-macros": "^3.1.0" }, @@ -9422,6 +9664,17 @@ "node": ">=8" } }, + "node_modules/doctrine": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/doctrine/-/doctrine-3.0.0.tgz", + "integrity": "sha512-yS+Q5i3hBf7GBkd4KG8a7eBNNWNGLTaEwwYWUijIYM7zrlYDM0BFXHjjPWlWZ1Rg7UaddZeIDmi9jF3HmqiQ2w==", + "dependencies": { + "esutils": "^2.0.2" + }, + "engines": { + "node": ">=6.0.0" + } + }, "node_modules/dom-converter": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/dom-converter/-/dom-converter-0.2.0.tgz", @@ -9624,7 +9877,6 @@ "version": "2.4.1", "resolved": "https://registry.npmjs.org/enquirer/-/enquirer-2.4.1.tgz", "integrity": "sha512-rRqJg/6gd538VHvR3PSrdRBb/1Vy2YfzHqzvbhGIQpDRKIa4FgV/54b5Q1xYSxOOwKvjXweS26E0Q+nAMwp2pQ==", - "license": "MIT", "dependencies": { "ansi-colors": "^4.1.1", "strip-ansi": "^6.0.1" @@ -9883,7 +10135,6 @@ "version": "4.0.1", "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz", "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==", - "license": "BSD-2-Clause", "bin": { "esparse": "bin/esparse.js", "esvalidate": "bin/esvalidate.js" @@ -12045,9 +12296,7 @@ "node_modules/js-tokens": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", - "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", - "dev": true, - "license": "MIT" + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==" }, "node_modules/js-yaml": { "version": "4.1.0", @@ -12062,15 +12311,14 @@ } }, "node_modules/jsesc": { - "version": "2.5.2", - "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-2.5.2.tgz", - "integrity": "sha512-OYu7XEzjkCQ3C5Ps3QIZsQfNpqoJyZZA99wd9aWd05NCtC5pWOkShK2mkL6HXQR6/Cy2lbNdPlZBpuQHXE63gA==", - "license": "MIT", + "version": "3.0.2", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.2.tgz", + "integrity": "sha512-xKqzzWXDttJuOcawBt4KnKHHIf5oQ/Cxax+0PWFG+DFDgHNAdi+TXECADI+RYiFUMmx8792xsMbbgXj4CwnP4g==", "bin": { "jsesc": "bin/jsesc" }, "engines": { - "node": ">=4" + "node": ">=6" } }, "node_modules/json-bigint": { @@ -13120,6 +13368,10 @@ "integrity": "sha512-ySViT69/76t8VhE1xXHK6Ch4NcDd26gx0MzKXLO+F7NOtnqH68d9zF94nT8ZWSxXh8ELOERsnJO/sWt1xZYw5A==", "license": "MIT" }, + "node_modules/module-docgen": { + "resolved": "tools/module-docgen", + "link": true + }, "node_modules/moment": { "version": "2.30.1", "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz", @@ -15089,12 +15341,9 @@ "license": "MIT" }, "node_modules/semver": { - "version": "7.6.0", - "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.0.tgz", - "integrity": "sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg==", - "dependencies": { - "lru-cache": "^6.0.0" - }, + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", "bin": { "semver": "bin/semver.js" }, @@ -15102,22 +15351,6 @@ "node": ">=10" } }, - "node_modules/semver/node_modules/lru-cache": { - "version": "6.0.0", - "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-6.0.0.tgz", - "integrity": "sha512-Jo6dJ04CmSjuznwJSS3pUeWmd/H0ffTlkXXgwZi+eq1UCmqQwCh+eLsYOYCwY991i2Fah4h1BEMCx4qThGbsiA==", - "dependencies": { - "yallist": "^4.0.0" - }, - "engines": { - "node": ">=10" - } - }, - "node_modules/semver/node_modules/yallist": { - "version": "4.0.0", - "resolved": "https://registry.npmjs.org/yallist/-/yallist-4.0.0.tgz", - "integrity": "sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==" - }, "node_modules/send": { "version": "0.19.0", "resolved": "https://registry.npmjs.org/send/-/send-0.19.0.tgz", @@ -15242,6 +15475,56 @@ "node": ">=8" } }, + "node_modules/sharp": { + "version": "0.33.5", + "resolved": "https://registry.npmjs.org/sharp/-/sharp-0.33.5.tgz", + "integrity": "sha512-haPVm1EkS9pgvHrQ/F3Xy+hgcuMV0Wm9vfIBSiwZ05k+xgb0PkBQpGsAA/oWdDobNaZTH5ppvHtzCFbnSEwHVw==", + "hasInstallScript": true, + "dependencies": { + "color": "^4.2.3", + "detect-libc": "^2.0.3", + "semver": "^7.6.3" + }, + "engines": { + "node": "^18.17.0 || ^20.3.0 || >=21.0.0" + }, + "funding": { + "url": "https://opencollective.com/libvips" + }, + "optionalDependencies": { + "@img/sharp-darwin-arm64": "0.33.5", + "@img/sharp-darwin-x64": "0.33.5", + "@img/sharp-libvips-darwin-arm64": "1.0.4", + "@img/sharp-libvips-darwin-x64": "1.0.4", + "@img/sharp-libvips-linux-arm": "1.0.5", + "@img/sharp-libvips-linux-arm64": "1.0.4", + "@img/sharp-libvips-linux-s390x": "1.0.4", + "@img/sharp-libvips-linux-x64": "1.0.4", + "@img/sharp-libvips-linuxmusl-arm64": "1.0.4", + "@img/sharp-libvips-linuxmusl-x64": "1.0.4", + "@img/sharp-linux-arm": "0.33.5", + "@img/sharp-linux-arm64": "0.33.5", + "@img/sharp-linux-s390x": "0.33.5", + "@img/sharp-linux-x64": "0.33.5", + "@img/sharp-linuxmusl-arm64": "0.33.5", + "@img/sharp-linuxmusl-x64": "0.33.5", + "@img/sharp-wasm32": "0.33.5", + "@img/sharp-win32-ia32": "0.33.5", + "@img/sharp-win32-x64": "0.33.5" + } + }, + "node_modules/sharp/node_modules/color": { + "version": "4.2.3", + "resolved": "https://registry.npmjs.org/color/-/color-4.2.3.tgz", + "integrity": "sha512-1rXeuUUiGGrykh+CeBdu5Ie7OJwinCgQY0bc7GCRxy5xVHy+moaqkpL/jqQq0MtQOeYcrqEz4abc5f0KtU7W4A==", + "dependencies": { + "color-convert": "^2.0.1", + "color-string": "^1.9.0" + }, + "engines": { + "node": ">=12.5.0" + } + }, "node_modules/shebang-command": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz", @@ -15612,10 +15895,6 @@ "node": ">= 0.8" } }, - "node_modules/strataparse": { - "resolved": "src/strataparse", - "link": true - }, "node_modules/streamsearch": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/streamsearch/-/streamsearch-1.1.0.tgz", @@ -16052,15 +16331,6 @@ "integrity": "sha512-XPaBkWQJdsf3pLKJV9p4qN/S+fm2Oj8AIPo1BTUhg5oxkvm9+SVEGFdhyOz7tTdUTfvxMiAs4sp6/eZO2Ew+pw==", "license": "MIT" }, - "node_modules/to-fast-properties": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-2.0.0.tgz", - "integrity": "sha512-/OaKK0xYrs3DmxRYqL/yDc+FxFUVYhDlXMhRmv3z915w2HF1tnN1omB354j8VUGO/hbRzyD6Y3sA7v7GS/ceog==", - "license": "MIT", - "engines": { - "node": ">=4" - } - }, "node_modules/to-regex-range": { "version": "5.0.1", "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", @@ -16842,6 +17112,14 @@ "integrity": "sha512-baNZyqaaLhyLVKm/DlvdW051MSgO6b8eVfIezl9E5PqWxFgzLm/wQntEW4zOytVburDEr0JlALEpdOFwvErLsg==", "license": "MIT" }, + "node_modules/word-wrap": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/word-wrap/-/word-wrap-1.2.5.tgz", + "integrity": "sha512-BN22B5eaMMI9UMtjrGd5g5eCYPpCPDUy0FJXbYsaT5zYxjFOckS53SQDE3pWkVoWpHXVb3BrYcEN4Twa55B5cA==", + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/wordwrap": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/wordwrap/-/wordwrap-1.0.0.tgz", @@ -17412,6 +17690,7 @@ }, "src/contextlink": { "version": "0.0.0", + "extraneous": true, "license": "AGPL-3.0-only", "devDependencies": { "mocha": "^10.2.0" @@ -17489,6 +17768,12 @@ "src/parsely": { "name": "@heyputer/parsely", "version": "1.0.0", + "extraneous": true, + "license": "AGPL-3.0-only" + }, + "src/parsers": { + "name": "@heyputer/parsers", + "version": "1.0.0", "license": "AGPL-3.0-only" }, "src/phoenix": { @@ -17594,6 +17879,7 @@ }, "src/strataparse": { "version": "0.0.0", + "extraneous": true, "license": "AGPL-3.0-only" }, "src/terminal": { @@ -17693,6 +17979,20 @@ "node": ">= 14.16" } }, + "tools/comment-writer": { + "version": "1.0.0", + "license": "AGPL-3.0-only", + "dependencies": { + "axios": "^1.7.8", + "console-table-printer": "^2.12.1", + "dedent": "^1.5.3", + "diff-match-patch": "^1.0.5", + "enquirer": "^2.4.1", + "js-levenshtein": "^1.1.6", + "word-wrap": "^1.2.5", + "yaml": "^2.4.5" + } + }, "tools/file-walker": { "version": "1.0.0", "license": "AGPL-3.0-only" @@ -17738,6 +18038,16 @@ "node": ">=18" } }, + "tools/module-docgen": { + "version": "1.0.0", + "license": "AGPL-3.0-only", + "dependencies": { + "@babel/parser": "^7.26.2", + "@babel/traverse": "^7.25.9", + "dedent": "^1.5.3", + "doctrine": "^3.0.0" + } + }, "tools/token-count-accuracy": { "version": "1.0.0", "license": "AGPL-3.0-only" diff --git a/package.json b/package.json index 613ccb2e9e..520f32b462 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "webpack-cli": "^5.1.1" }, "scripts": { - "test": "npx mocha src/phoenix/test src/contextlink/test && node src/backend/tools/test", + "test": "npx mocha src/phoenix/test && node src/backend/tools/test", "start=gui": "nodemon --exec \"node dev-server.js\" ", "start": "node ./tools/run-selfhosted.js", "build": "cd src/gui; node ./build.js", @@ -50,6 +50,7 @@ "javascript-time-ago": "^2.5.11", "json-colorizer": "^3.0.1", "open": "^10.1.0", + "sharp": "^0.33.5", "simple-git": "^3.25.0", "string-template": "^1.0.0", "uuid": "^9.0.1" diff --git a/src/backend/doc/contributors/index.md b/src/backend/CONTRIBUTING.md similarity index 74% rename from src/backend/doc/contributors/index.md rename to src/backend/CONTRIBUTING.md index 381aeef0df..a3101335da 100644 --- a/src/backend/doc/contributors/index.md +++ b/src/backend/CONTRIBUTING.md @@ -1,18 +1,22 @@ # Contributing to Puter's Backend +## File Structure + + + ## Architecture -- [boot sequence](./boot-sequence.md) -- [modules and services](./modules.md) +- [boot sequence](./doc/contributors/boot-sequence.md) +- [modules and services](./doc/contributors/modules.md) ## Features -- [protected apps](../features/protected-apps.md) -- [service scripts](../features/service-scripts.md) +- [protected apps](./doc/features/protected-apps.md) +- [service scripts](./doc/features/service-scripts.md) ## Lists of Things -- [list of permissions](../lists-of-things/list-of-permissions.md) +- [list of permissions](./doc/lists-of-things/list-of-permissions.md) ## Code-First Approach @@ -20,21 +24,21 @@ If you prefer to understand a system by looking at the first files which are invoked and starting from there, here's a handy list! -- [Kernel](../../src/Kernel.js), despite its intimidating name, is a +- [Kernel](./src/Kernel.js), despite its intimidating name, is a relatively simple (< 200 LOC) class which loads the modules (modules register services), and then starts all the services. -- [RuntimeEnvironment](../../src/boot/RuntimeEnvironment.js) +- [RuntimeEnvironment](./src/boot/RuntimeEnvironment.js) sets the configuration and runtime directories. It's invoked by Kernel. - The default setup for running a self-hosted Puter loads these modules: - - [CoreModule](../../src/CoreModule.js) - - [DatabaseModule](../../src/DatabaseModule.js) - - [LocalDiskStorageModule](../../src/LocalDiskStorageModule.js) + - [CoreModule](./src/CoreModule.js) + - [DatabaseModule](./src/DatabaseModule.js) + - [LocalDiskStorageModule](./src/LocalDiskStorageModule.js) - HTTP endpoints are registered with - [WebServerService](../../src/services/WebServerService.js) + [WebServerService](./src/services/WebServerService.js) by these services: - - [ServeGUIService](../../src/services/ServeGUIService.js) - - [PuterAPIService](../../src/services/PuterAPIService.js) - - [FilesystemAPIService](../../src/services/FilesystemAPIService.js) + - [ServeGUIService](./src/services/ServeGUIService.js) + - [PuterAPIService](./src/services/PuterAPIService.js) + - [FilesystemAPIService](./src/services/FilesystemAPIService.js) ## Development Philosophies @@ -71,7 +75,7 @@ doing the useless work that reveals what the useful work is. ## Underlying Constructs -- [putility's README.md](../../packages/putility/README.md) +- [putility's README.md](../putility/README.md) - Whenever you see `AdvancedBase`, that's from here - Many things in backend extend this. Anything that doesn't only doesn't because it was written before `AdvancedBase` existed. diff --git a/src/backend/doc/Kernel.md b/src/backend/doc/Kernel.md new file mode 100644 index 0000000000..0cb15ef05d --- /dev/null +++ b/src/backend/doc/Kernel.md @@ -0,0 +1,65 @@ +# Puter Kernel Documentation + +## Overview + +The **Puter Kernel** is the core runtime component of the Puter system. It provides the foundational infrastructure for: + +- Initializing the runtime environment +- Managing internal and external modules (extensions) +- Setting up and booting core services +- Configuring logging and debugging utilities +- Integrating with third-party modules and performing dependency installs at runtime + +This kernel is responsible for orchestrating the startup sequence and ensuring that all necessary services, modules, and environmental configurations are properly loaded before the application enters its operational state. + +--- + +## Features + +1. **Modular Architecture**: + The Kernel supports both internal and external modules: + - **Internal Modules**: Provided to Kernel by an initializing script, such + as `tools/run-selfhosted.js`, via the `add_module()` method. + - **External Modules**: Discovered in configured module directories and installed + dynamically. This includes resolving and executing `package.json` entries and + running `npm install` as needed. + +2. **Service Container & Registry**: + The Kernel initializes a service container that manages a wide range of services. Services can: + - Register modules + - Initialize dependencies + - Emit lifecycle events (`boot.consolidation`, `boot.activation`, `boot.ready`) to + orchestrate a stable and consistent environment. + +3. **Runtime Environment Setup**: + The Kernel sets up a `RuntimeEnvironment` to determine configuration paths and environment parameters. It also provides global helpers like `kv` for key-value storage and `cl` for simplified console logging. + +4. **Logging and Debugging**: + Uses a temporary `BootLogger` for the initialization phase until LogService is + initialized, at which point it will replace the boot logger. Debugging features + (`ll`, `xtra_log`) are enabled in development environments for convenience. + +## Initialization & Boot Process + +1. **Constructor**: + When a Kernel instance is created, it sets up basic parameters, initializes an empty + module list, and prepares `useapi()` integration. + +2. **Booting**: + The `boot()` method: + - Parses CLI arguments using `yargs`. + - Calls `_runtime_init()` to set up the `RuntimeEnvironment` and boot logger. + - Initializes global debugging/logging utilities. + - Sets up the service container (usually called `services`c instance of **Container**). + - Invokes module installation and service bootstrapping processes. + +3. **Module Installation**: + Internal modules are registered and installed first. + External modules are discovered, packaged, installed, and their code is executed. + External modules are given a special context with access to `useapi()`, a dynamic + import mechanism for Puter modules and extensions. + +4. **Service Bootstrapping**: + After modules and extensions are installed, services are initialized and activated. + For more information about how this works, see [boot-sequence.md](./contributors/boot-sequence.md). + diff --git a/src/backend/exports.js b/src/backend/exports.js index 0ed27f9914..9ea422cc54 100644 --- a/src/backend/exports.js +++ b/src/backend/exports.js @@ -21,13 +21,17 @@ const { Kernel } = require("./src/Kernel.js"); const DatabaseModule = require("./src/DatabaseModule.js"); const LocalDiskStorageModule = require("./src/LocalDiskStorageModule.js"); const SelfHostedModule = require("./src/modules/selfhosted/SelfHostedModule.js"); -const PuterDriversModule = require("./src/PuterDriversModule.js"); const { testlaunch } = require("./src/index.js"); const BaseService = require("./src/services/BaseService.js"); const { Context } = require("./src/util/context.js"); const { TestDriversModule } = require("./src/modules/test-drivers/TestDriversModule.js"); const { PuterAIModule } = require("./src/modules/puterai/PuterAIModule.js"); const { BroadcastModule } = require("./src/modules/broadcast/BroadcastModule.js"); +const { WebModule } = require("./src/modules/web/WebModule.js"); +const { Core2Module } = require("./src/modules/core/Core2Module.js"); +const { TemplateModule } = require("./src/modules/template/TemplateModule.js"); +const { PuterFSModule } = require("./src/modules/puterfs/PuterFSModule.js"); +const { PerfMonModule } = require("./src/modules/perfmon/PerfMonModule.js"); module.exports = { @@ -42,14 +46,25 @@ module.exports = { Context, Kernel, + + EssentialModules: [ + Core2Module, + PuterFSModule, + CoreModule, + WebModule, + TemplateModule, + ], // Pre-built modules CoreModule, + WebModule, DatabaseModule, - PuterDriversModule, LocalDiskStorageModule, SelfHostedModule, TestDriversModule, PuterAIModule, BroadcastModule, + + // Development modules + PerfMonModule, }; diff --git a/src/backend/src/CoreModule.js b/src/backend/src/CoreModule.js index 7a4fb7690d..31f2fc37b8 100644 --- a/src/backend/src/CoreModule.js +++ b/src/backend/src/CoreModule.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"claude"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -23,6 +24,16 @@ const { ProtectedAppES } = require("./om/entitystorage/ProtectedAppES"); const { Context } = require('./util/context'); + +/** + * Core module for the Puter platform that includes essential services including + * authentication, filesystems, rate limiting, permissions, and various API endpoints. + * + * This is a monolithic module. Incrementally, services should be migrated to + * Core2Module and other modules instead. Core2Module has a smaller scope, and each + * new module will be a cohesive concern. Once CoreModule is empty, it will be removed + * and Core2Module will take on its name. + */ class CoreModule extends AdvancedBase { dirname () { return __dirname; } async install (context) { @@ -33,11 +44,16 @@ class CoreModule extends AdvancedBase { await install({ services, app, useapi, modapi }); } - // Some services were created before the BaseService - // class existed. They don't listen to the init event - // and the order in which they're instantiated matters. - // They all need to be installed after the init event - // is dispatched, so they get a separate install method. + /** + * Installs legacy services that don't extend BaseService and require special handling. + * These services were created before the BaseService class existed and don't listen + * to the init event. They need to be installed after the init event is dispatched + * due to initialization order dependencies. + * + * @param {Object} context - The context object containing service references + * @param {Object} context.services - Service registry for registering legacy services + * @returns {Promise} Resolves when legacy services are installed + */ async install_legacy (context) { const services = context.get('services'); await install_legacy({ services }); @@ -52,6 +68,9 @@ module.exports = CoreModule; const install = async ({ services, app, useapi, modapi }) => { const config = require('./config'); + + // === LIBRARIES === + useapi.withuse(() => { def('Service', require('./services/BaseService')); def('Module', AdvancedBase); @@ -68,7 +87,6 @@ const install = async ({ services, app, useapi, modapi }) => { def('core.config', config); }); - // === LIBRARIES === useapi.withuse(() => { const ArrayUtil = require('./libraries/ArrayUtil'); services.registerService('util-array', ArrayUtil); @@ -82,16 +100,11 @@ const install = async ({ services, app, useapi, modapi }) => { // === SERVICES === // /!\ IMPORTANT /!\ - // For new services, put the import immediate above the + // For new services, put the import immediately above the // call to services.registerService. We'll clean this up // in a future PR. - const { LogService } = require('./services/runtime-analysis/LogService'); - const { PagerService } = require('./services/runtime-analysis/PagerService'); - const { AlarmService } = require('./services/runtime-analysis/AlarmService'); - const { ErrorService } = require('./services/runtime-analysis/ErrorService'); const { CommandService } = require('./services/CommandService'); - const { ExpectationService } = require('./services/runtime-analysis/ExpectationService'); const { HTTPThumbnailService } = require('./services/thumbnails/HTTPThumbnailService'); const { PureJSThumbnailService } = require('./services/thumbnails/PureJSThumbnailService'); const { NAPIThumbnailService } = require('./services/thumbnails/NAPIThumbnailService'); @@ -124,12 +137,10 @@ const install = async ({ services, app, useapi, modapi }) => { const { ESBuilder } = require('./om/entitystorage/ESBuilder'); const { Eq, Or } = require('./om/query/query'); const { TrackSpendingService } = require('./services/TrackSpendingService'); - const { ServerHealthService } = require('./services/runtime-analysis/ServerHealthService'); const { MakeProdDebuggingLessAwfulService } = require('./services/MakeProdDebuggingLessAwfulService'); const { ConfigurableCountingService } = require('./services/ConfigurableCountingService'); const { FSLockService } = require('./services/fs/FSLockService'); const { StrategizedService } = require('./services/StrategizedService'); - const WebServerService = require('./services/WebServerService'); const FilesystemAPIService = require('./services/FilesystemAPIService'); const ServeGUIService = require('./services/ServeGUIService'); const PuterAPIService = require('./services/PuterAPIService'); @@ -140,17 +151,10 @@ const install = async ({ services, app, useapi, modapi }) => { // === Services which extend BaseService === services.registerService('system-validation', SystemValidationService); - services.registerService('server-health', ServerHealthService); - services.registerService('log-service', LogService); services.registerService('commands', CommandService); - services.registerService('web-server', WebServerService, { app }); services.registerService('__api-filesystem', FilesystemAPIService); services.registerService('__api', PuterAPIService); services.registerService('__gui', ServeGUIService); - services.registerService('expectations', ExpectationService); - services.registerService('pager', PagerService); - services.registerService('alarm', AlarmService); - services.registerService('error-service', ErrorService); services.registerService('registry', RegistryService); services.registerService('__registrant', RegistrantService); services.registerService('fslock', FSLockService); @@ -187,9 +191,6 @@ const install = async ({ services, app, useapi, modapi }) => { ]), }); - const { ParameterService } = require('./services/ParameterService'); - services.registerService('params', ParameterService); - const { InformationService } = require('./services/information/InformationService'); services.registerService('information', InformationService) @@ -316,9 +317,6 @@ const install = async ({ services, app, useapi, modapi }) => { const { PermissionAPIService } = require('./services/PermissionAPIService'); services.registerService('__permission-api', PermissionAPIService); - const { MountpointService } = require('./services/MountpointService'); - services.registerService('mountpoint', MountpointService); - const { AnomalyService } = require('./services/AnomalyService'); services.registerService('anomaly', AnomalyService); @@ -351,29 +349,32 @@ const install = async ({ services, app, useapi, modapi }) => { const { ReferralCodeService } = require('./services/ReferralCodeService'); services.registerService('referral-code', ReferralCodeService); + + const { UserService } = require('./services/UserService'); + services.registerService('user', UserService); + + const { WSPushService } = require('./services/WSPushService'); + services.registerService('__event-push-ws', WSPushService); + + const { AppIconService } = require('./services/AppIconService'); + services.registerService('app-icon', AppIconService); } const install_legacy = async ({ services }) => { - const { ProcessEventService } = require('./services/runtime-analysis/ProcessEventService'); - // const { FilesystemService } = require('./filesystem/FilesystemService'); const PerformanceMonitor = require('./monitor/PerformanceMonitor'); const { OperationTraceService } = require('./services/OperationTraceService'); - const { WSPushService } = require('./services/WSPushService'); const { ClientOperationService } = require('./services/ClientOperationService'); const { EngPortalService } = require('./services/EngPortalService'); const { AppInformationService } = require('./services/AppInformationService'); const { FileCacheService } = require('./services/file-cache/FileCacheService'); // === Services which do not yet extend BaseService === - services.registerService('process-event', ProcessEventService); // services.registerService('filesystem', FilesystemService); services.registerService('operationTrace', OperationTraceService); - services.registerService('__event-push-ws', WSPushService); services.registerService('file-cache', FileCacheService); services.registerService('client-operation', ClientOperationService); services.registerService('app-information', AppInformationService); services.registerService('engineering-portal', EngPortalService); - // TODO: add to here: ResourceService and DatabaseFSEntryService // This singleton was made before services existed, // so we have to pass that to it manually diff --git a/src/backend/src/Extension.js b/src/backend/src/Extension.js index 6b2d4ade60..e3601d2b91 100644 --- a/src/backend/src/Extension.js +++ b/src/backend/src/Extension.js @@ -3,6 +3,10 @@ const EmitterFeature = require("@heyputer/putility/src/features/EmitterFeature") const { Context } = require("./util/context"); const { ExtensionServiceState } = require("./ExtensionService"); +/** + * This class creates the `extension` global that is seem by Puter backend + * extensions. + */ class Extension extends AdvancedBase { static FEATURES = [ EmitterFeature({ @@ -24,6 +28,9 @@ class Extension extends AdvancedBase { console.log('Example method called by an extension.'); } + /** + * This will get a database instance from the default service. + */ get db () { const db = this.service.values.get('db'); if ( ! db ) { @@ -35,6 +42,12 @@ class Extension extends AdvancedBase { return db; } + /** + * This will create a GET endpoint on the default service. + * @param {*} path - route for the endpoint + * @param {*} handler - function to handle the endpoint + * @param {*} options - options like noauth (bool) and mw (array) + */ get (path, handler, options) { // this extension will have a default service this.ensure_service_(); @@ -51,6 +64,12 @@ class Extension extends AdvancedBase { }); } + /** + * This will create a POST endpoint on the default service. + * @param {*} path - route for the endpoint + * @param {*} handler - function to handle the endpoint + * @param {*} options - options like noauth (bool) and mw (array) + */ post (path, handler, options) { // this extension will have a default service this.ensure_service_(); @@ -67,6 +86,13 @@ class Extension extends AdvancedBase { }); } + /** + * This method will create the "default service" for an extension. + * This is specifically for Puter extensions that do not define their + * own service classes. + * + * @returns {void} + */ ensure_service_ () { if ( this.service ) { return; diff --git a/src/backend/src/ExtensionService.js b/src/backend/src/ExtensionService.js index 7ea96d12f1..8c867a9ce7 100644 --- a/src/backend/src/ExtensionService.js +++ b/src/backend/src/ExtensionService.js @@ -5,6 +5,11 @@ const configurable_auth = require("./middleware/configurable_auth"); const { Context } = require("./util/context"); const { DB_READ, DB_WRITE } = require("./services/database/consts"); +/** + * State shared with the default service and the `extension` global so that + * methods on `extension` can register routes (and make other changes in the + * future) to the default service. + */ class ExtensionServiceState extends AdvancedBase { constructor (...a) { super(...a); @@ -75,6 +80,7 @@ class ExtensionService extends BaseService { } ['__on_install.routes'] (_, { app }) { + if ( ! this.state ) debugger; for ( const endpoint of this.state.endpoints_ ) { endpoint.attach(app); } diff --git a/src/backend/src/Kernel.js b/src/backend/src/Kernel.js index 1f7a5ecb7c..f99cafd31c 100644 --- a/src/backend/src/Kernel.js +++ b/src/backend/src/Kernel.js @@ -16,7 +16,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -const { AdvancedBase } = require("@heyputer/putility"); +const { AdvancedBase, libs } = require("@heyputer/putility"); const { Context } = require('./util/context'); const BaseService = require("./services/BaseService"); const useapi = require('useapi'); @@ -25,7 +25,8 @@ const { hideBin } = require('yargs/helpers'); const { Extension } = require("./Extension"); const { ExtensionModule } = require("./ExtensionModule"); const { spawn } = require("node:child_process"); -const { quot } = require("./util/strutil"); + +const { quot } = libs.string; class Kernel extends AdvancedBase { constructor ({ entry_path } = {}) { @@ -78,8 +79,6 @@ class Kernel extends AdvancedBase { this._runtime_init({ args }); - // const express = require('express') - // const app = express(); const config = require('./config'); globalThis.ll = o => o; @@ -101,18 +100,11 @@ class Kernel extends AdvancedBase { const { consoleLogManager } = require('./util/consolelog'); consoleLogManager.initialize_proxy_methods(); - // TODO: temporary dependency inversion; requires moving: - // - rm, so we can move mv - // - mv, so we can move mkdir - // - generate_default_fsentries, so we can move mkdir - // - mkdir, which needs an fs provider - // === START: Initialize Service Registry === const { Container } = require('./services/Container'); const services = new Container({ logger: this.bootLogger }); this.services = services; - // app.set('services', services); const root_context = Context.create({ environment: this.environment, @@ -130,7 +122,6 @@ class Kernel extends AdvancedBase { }); - // Error.stackTraceLimit = Infinity; Error.stackTraceLimit = 200; } @@ -238,104 +229,93 @@ class Kernel extends AdvancedBase { } const mod_dirnames = fs.readdirSync(mods_dirpath); for ( const mod_dirname of mod_dirnames ) { - let mod_path = path_.join(mods_dirpath, mod_dirname); + await this.install_extern_mod_({ + mod_install_root_context, + mod_dirname, + mod_path: path_.join(mods_dirpath, mod_dirname), + }); + } + } + } + + async install_extern_mod_({ + mod_install_root_context, + mod_dirname, + mod_path, + }) { + const path_ = require('path'); + const fs = require('fs'); - let stat = fs.lstatSync(mod_path); - while ( stat.isSymbolicLink() ) { - mod_path = fs.readlinkSync(mod_path); - stat = fs.lstatSync(mod_path); - } + let stat = fs.lstatSync(mod_path); + while ( stat.isSymbolicLink() ) { + mod_path = fs.readlinkSync(mod_path); + stat = fs.lstatSync(mod_path); + } - if ( ! stat.isDirectory() && !(mod_dirname.endsWith('.js')) ) { - continue; - } - - const mod_name = path_.parse(mod_path).name; - const mod_package_dir = `mod_packages/${mod_name}`; - fs.mkdirSync(mod_package_dir); - - if ( ! stat.isDirectory() ) { - this.create_mod_package_json(mod_package_dir, { - name: mod_name, - }); - fs.copyFileSync(mod_path, path_.join(mod_package_dir, 'main.js')); - } else { - if ( ! fs.existsSync(path_.join(mod_path, 'package.json')) ) { - // Expect main.js or index.js to exist - const options = ['main.js', 'index.js']; - let entry_file = null; - for ( const option of options ) { - if ( fs.existsSync(path_.join(mod_path, option)) ) { - entry_file = option; - break; - } - } - if ( ! entry_file ) { - // If directory is empty, we'll just skip it - if ( fs.readdirSync(mod_path).length === 0 ) { - this.bootLogger.warn(`Empty mod directory ${quot(mod_path)}; skipping...`); - continue; - } - - // Other wise, we'll throw an error - this.bootLogger.error(`Expected main.js or index.js in ${quot(mod_path)}`); - if ( ! process.env.SKIP_INVALID_MODS ) { - this.bootLogger.error(`Set SKIP_INVALID_MODS=1 (environment variable) to run anyway.`); - process.exit(1); - } else { - continue; - } - } - - this.create_mod_package_json(mod_package_dir, { - name: mod_name, - entry: entry_file, - }); - } - fs.cpSync(mod_path, mod_package_dir, { - recursive: true, - }); - } - - const mod_require_dir = path_.join(process.cwd(), mod_package_dir); - - await this.run_npm_install(mod_require_dir); - - const mod = new ExtensionModule(); - mod.extension = new Extension(); - - // This is where the module gets the 'use' and 'def' globals - await this.useapi.awithuse(async () => { - // This is where the module gets the 'extension' global - await useapi.aglobalwith({ - extension: mod.extension, - }, async () => { - const maybe_promise = require(mod_require_dir); - if ( maybe_promise && maybe_promise instanceof Promise ) { - await maybe_promise; - } - }); - }); + // Mod must be a directory or javascript file + if ( ! stat.isDirectory() && !(mod_path.endsWith('.js')) ) { + return; + } + + const mod_name = path_.parse(mod_path).name; + const mod_package_dir = `mod_packages/${mod_name}`; + fs.mkdirSync(mod_package_dir); + + if ( ! stat.isDirectory() ) { + this.create_mod_package_json(mod_package_dir, { + name: mod_name, + entry: 'main.js' + }); + fs.copyFileSync(mod_path, path_.join(mod_package_dir, 'main.js')); + } else { + // If directory is empty, we'll just skip it + if ( fs.readdirSync(mod_path).length === 0 ) { + this.bootLogger.warn(`Empty mod directory ${quot(mod_path)}; skipping...`); + return; + } - const mod_context = this._create_mod_context(mod_install_root_context, { - name: mod_dirname, - ['module']: mod, - external: true, - mod_path, - }); - - // TODO: DRY `awithuse` and `aglobalwith` with above - await this.useapi.awithuse(async () => { - await useapi.aglobalwith({ - extension: mod.extension, - }, async () => { - // This is where the 'install' event gets triggered - await mod.install(mod_context); - }); + // Create package.json if it doesn't exist + if ( ! fs.existsSync(path_.join(mod_path, 'package.json')) ) { + this.create_mod_package_json(mod_package_dir, { + name: mod_name, }); } + + // Copy mod contents to `/mod_packages` + fs.cpSync(mod_path, mod_package_dir, { + recursive: true, + }); } - } + + const mod_require_dir = path_.join(process.cwd(), mod_package_dir); + + await this.run_npm_install(mod_require_dir); + + const mod = new ExtensionModule(); + mod.extension = new Extension(); + + const mod_context = this._create_mod_context(mod_install_root_context, { + name: mod_dirname, + ['module']: mod, + external: true, + mod_path, + }); + + // This is where the module gets the 'use' and 'def' globals + await this.useapi.awithuse(async () => { + // This is where the module gets the 'extension' global + await useapi.aglobalwith({ + extension: mod.extension, + }, async () => { + const maybe_promise = require(mod_require_dir); + if ( maybe_promise && maybe_promise instanceof Promise ) { + await maybe_promise; + } + // This is where the 'install' event gets triggered + await mod.install(mod_context); + }); + }); + }; _create_mod_context (parent, options) { const path_ = require('path'); @@ -380,6 +360,30 @@ class Kernel extends AdvancedBase { const fs = require('fs'); const path_ = require('path'); + // Expect main.js or index.js to exist + const options = ['main.js', 'index.js']; + + // If no entry specified, find file with conventional name + if ( ! entry ) { + for ( const option of options ) { + if ( fs.existsSync(path_.join(mod_path, option)) ) { + entry = option; + break; + } + } + } + + // If no entry specified or found, skip or error + if ( ! entry ) { + this.bootLogger.error(`Expected main.js or index.js in ${quot(mod_path)}`); + if ( ! process.env.SKIP_INVALID_MODS ) { + this.bootLogger.error(`Set SKIP_INVALID_MODS=1 (environment variable) to run anyway.`); + process.exit(1); + } else { + return; + } + } + const data = JSON.stringify({ name, version: '1.0.0', diff --git a/src/backend/src/PuterDriversModule.js b/src/backend/src/PuterDriversModule.js deleted file mode 100644 index 7cb6e55e52..0000000000 --- a/src/backend/src/PuterDriversModule.js +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright (C) 2024 Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require("@heyputer/putility"); - -class PuterDriversModule extends AdvancedBase { - async install () {} - async install_legacy (context) { - const services = context.get('services'); - - const { DriverService } = require("./services/drivers/DriverService"); - services.registerService('driver', DriverService); - } -} - -module.exports = PuterDriversModule; diff --git a/src/backend/src/api/APIError.js b/src/backend/src/api/APIError.js index d8c678dc91..eb16f341d4 100644 --- a/src/backend/src/api/APIError.js +++ b/src/backend/src/api/APIError.js @@ -17,7 +17,7 @@ * along with this program. If not, see . */ const { URLSearchParams } = require("node:url"); -const { quot } = require("../util/strutil"); +const { quot } = require('@heyputer/putility').libs.string; /** * APIError represents an error that can be sent to the client. @@ -324,36 +324,6 @@ module.exports = class APIError { message: () => 'Invalid token.', }, - // drivers - 'interface_not_found': { - status: 404, - message: ({ interface_name }) => `Interface not found: ${quot(interface_name)}`, - }, - 'no_implementation_available': { - status: 502, - message: ({ - iface, - interface_name, - driver - }) => `No implementation available for ` + - (iface ?? interface_name) ? 'interface' : 'driver' + - ' ' + quot(iface ?? interface_name ?? driver) + '.', - }, - 'method_not_found': { - status: 404, - message: ({ interface_name, method_name }) => `Method not found: ${quot(method_name)} on interface ${quot(interface_name)}`, - }, - 'missing_required_argument': { - status: 400, - message: ({ interface_name, method_name, arg_name }) => - `Missing required argument ${quot(arg_name)} for method ${quot(method_name)} on interface ${quot(interface_name)}`, - }, - 'argument_consolidation_failed': { - status: 400, - message: ({ interface_name, method_name, arg_name, message }) => - `Failed to parse or process argument ${quot(arg_name)} for method ${quot(method_name)} on interface ${quot(interface_name)}: ${message}`, - }, - // SLA 'rate_limit_exceeded': { status: 429, @@ -505,18 +475,6 @@ module.exports = class APIError { status: 400, message: 'Incorrect or missing anti-CSRF token.', }, - - // Chat - // TODO: specifying these errors here might be a violation - // of separation of concerns. Services could register their - // own errors with an error registry. - 'max_tokens_exceeded': { - status: 400, - message: ({ input_tokens, max_tokens }) => - `Input exceeds maximum token count. ` + - `Input has ${input_tokens} tokens, ` + - `but the maximum is ${max_tokens}.`, - }, }; /** diff --git a/src/backend/src/api/eggspress.js b/src/backend/src/api/eggspress.js index fca0bbf98c..5840ef9fd8 100644 --- a/src/backend/src/api/eggspress.js +++ b/src/backend/src/api/eggspress.js @@ -1,198 +1,2 @@ -/* - * Copyright (C) 2024 Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const express = require('express'); -const multer = require('multer'); -const multest = require('@heyputer/multest'); -const api_error_handler = require('../api/api_error_handler.js'); - -const fsBeforeMW = require('../middleware/fs'); -const APIError = require('./APIError.js'); -const { Context } = require('../util/context.js'); - -/** - * eggspress() is a factory function for creating express routers. - * - * @param {*} route the route to the router - * @param {*} settings the settings for the router. The following - * properties are supported: - * - auth: whether or not to use the auth middleware - * - fs: whether or not to use the fs middleware - * - json: whether or not to use the json middleware - * - customArgs: custom arguments to pass to the router - * - allowedMethods: the allowed HTTP methods - * @param {*} handler the handler for the router - * @returns {express.Router} the router - */ -module.exports = function eggspress (route, settings, handler) { - const router = express.Router(); - const mw = []; - const afterMW = []; - - // These flags enable specific middleware. - if ( settings.abuse ) mw.push(require('../middleware/abuse')(settings.abuse)); - if ( settings.auth ) mw.push(require('../middleware/auth')); - if ( settings.auth2 ) mw.push(require('../middleware/auth2')); - if ( settings.fs ) { - mw.push(fsBeforeMW); - } - if ( settings.verified ) mw.push(require('../middleware/verified')); - if ( settings.json ) mw.push(express.json()); - - // The `files` setting is an array of strings. Each string is the name - // of a multipart field that contains files. `multer` is used to parse - // the multipart request and store the files in `req.files`. - if ( settings.files ) { - for ( const key of settings.files ) { - mw.push(multer().array(key)); - } - } - - if ( settings.multest ) { - mw.push(multest()); - } - - // The `multipart_jsons` setting is an array of strings. Each string - // is the name of a multipart field that contains JSON. This middleware - // parses the JSON in each field and stores the result in `req.body`. - if ( settings.multipart_jsons ) { - for ( const key of settings.multipart_jsons ) { - mw.push((req, res, next) => { - try { - if ( ! Array.isArray(req.body[key]) ) { - req.body[key] = [JSON.parse(req.body[key])]; - } else { - req.body[key] = req.body[key].map(JSON.parse); - } - } catch (e) { - return res.status(400).send({ - error: { - message: `Invalid JSON in multipart field ${key}` - } - }); - } - next(); - }); - } - } - - // The `alias` setting is an object. Each key is the name of a - // parameter. Each value is the name of a parameter that should - // be aliased to the key. - if ( settings.alias ) { - for ( const alias in settings.alias ) { - const target = settings.alias[alias]; - mw.push((req, res, next) => { - const values = req.method === 'GET' ? req.query : req.body; - if ( values[alias] ) { - values[target] = values[alias]; - } - next(); - }); - } - } - - // The `parameters` setting is an object. Each key is the name of a - // parameter. Each value is a `Param` object. The `Param` object - // specifies how to validate the parameter. - if ( settings.parameters ) { - for ( const key in settings.parameters ) { - const param = settings.parameters[key]; - mw.push(async (req, res, next) => { - if ( ! req.values ) req.values = {}; - - const values = req.method === 'GET' ? req.query : req.body; - const getParam = (key) => values[key]; - try { - const result = await param.consolidate({ req, getParam }); - req.values[key] = result; - } catch (e) { - api_error_handler(e, req, res, next); - return; - } - next(); - }); - } - } - - // what if I wanted to pass arguments to, for example, `json`? - if ( settings.customArgs ) mw.push(settings.customArgs); - - if ( settings.alarm_timeout ) { - mw.push((req, res, next) => { - setTimeout(() => { - if ( ! res.headersSent ) { - const log = req.services.get('log-service').create('eggspress:timeout'); - const errors = req.services.get('error-service').create(log); - let id = Array.isArray(route) ? route[0] : route; - id = id.replace(/\//g, '_'); - errors.report(id, { - source: new Error('Response timed out.'), - message: 'Response timed out.', - trace: true, - alarm: true, - }); - } - }, settings.alarm_timeout); - next(); - }); - } - - if ( settings.response_timeout ) { - mw.push((req, res, next) => { - setTimeout(() => { - if ( ! res.headersSent ) { - api_error_handler(APIError.create('response_timeout'), req, res, next); - } - }, settings.response_timeout); - next(); - }); - } - - if ( settings.mw ) mw.push(...settings.mw); - - const errorHandledHandler = async function (req, res, next) { - if ( settings.subdomain ) { - if ( require('../helpers').subdomain(req) !== settings.subdomain ) { - return next(); - } - } - try { - const expected_ctx = res.locals.ctx; - const received_ctx = Context.get(undefined, { allow_fallback: true }); - - if ( expected_ctx != received_ctx ) { - await expected_ctx.arun(async () => { - await handler(req, res, next); - }); - } else await handler(req, res, next); - } catch (e) { - api_error_handler(e, req, res, next); - } - }; - - if ( settings.allowedMethods.includes('GET') ) { - router.get(route, ...mw, errorHandledHandler, ...afterMW); - } - - if ( settings.allowedMethods.includes('POST') ) { - router.post(route, ...mw, errorHandledHandler, ...afterMW); - } - - return router; -} \ No newline at end of file +// This file is a legacy alias +module.exports = require('../modules/web/lib/eggspress.js'); diff --git a/src/backend/src/api/filesystem/UserParam.js b/src/backend/src/api/filesystem/UserParam.js index 563824a8f2..75ff1fcf44 100644 --- a/src/backend/src/api/filesystem/UserParam.js +++ b/src/backend/src/api/filesystem/UserParam.js @@ -17,9 +17,6 @@ * along with this program. If not, see . */ module.exports = class UserParam { - constructor () { - // - } consolidate ({ req }) { return req.user; } diff --git a/src/backend/src/boot/RuntimeEnvironment.js b/src/backend/src/boot/RuntimeEnvironment.js index 4d30eb1316..09a88119b0 100644 --- a/src/backend/src/boot/RuntimeEnvironment.js +++ b/src/backend/src/boot/RuntimeEnvironment.js @@ -17,7 +17,7 @@ * along with this program. If not, see . */ const { AdvancedBase } = require("@heyputer/putility"); -const { quot } = require("../util/strutil"); +const { quot } = require('@heyputer/putility').libs.string; const { TechnicalError } = require("../errors/TechnicalError"); const { print_error_help } = require("../errors/error_help_details"); const default_config = require("./default_config"); @@ -93,8 +93,7 @@ const path_checks = ({ logger }) => ({ fs, path_ }) => ({ throw new Error(`No valid config file found in path: ${path}`); }, env_not_set: name => () => { - if ( process.env[name] ) return false; - return true; + return ! process.env[name]; } }); @@ -233,18 +232,14 @@ class RuntimeEnvironment extends AdvancedBase { ] ); + // Note: there used to be a 'mods_path_entry' here too + // but it was never used const pwd_path_entry = this.get_first_suitable_path_( { pathFor: 'working directory' }, this.runtime_paths, [ this.path_checks.require_write_permission ] ); - const mods_path_entry = this.get_first_suitable_path_( - { pathFor: 'mods', optional: true }, - this.mod_paths, - [ this.path_checks.require_read_permission ], - ); - process.chdir(pwd_path_entry.path); // Check for a valid config file in the config path @@ -287,9 +282,8 @@ class RuntimeEnvironment extends AdvancedBase { ); const config_values = JSON.parse(config_raw); for ( const k in generated_values ) { - if ( config_values[k] ) { - generated_values[k] = config_values[k]; - } + if ( ! config_values[k] ) continue; + generated_values[k] = config_values[k]; } } } @@ -334,9 +328,6 @@ class RuntimeEnvironment extends AdvancedBase { throw new Error('config_name is required'); } this.logger.info(hl(`config name`) + ` ${quot(config.config_name)}`); - // console.log(config.services); - // console.log(Object.keys(config.services)); - // console.log({ ...config.services }); const mod_paths = []; environment.mod_paths = mod_paths; @@ -363,12 +354,13 @@ class RuntimeEnvironment extends AdvancedBase { } get_first_suitable_path_ (meta, paths, last_checks) { - iter_paths: for ( const entry of paths ) { const checks = [...(entry.checks ?? []), ...last_checks]; this.logger.info( `Checking path ${quot(entry.label ?? entry.path)} for ${meta.pathFor}...` ); + + let checks_pass = true; for ( const check of checks ) { this.logger.info( `-> doing ${quot(check.name)} on path ${quot(entry.path)}...` @@ -378,9 +370,12 @@ class RuntimeEnvironment extends AdvancedBase { this.logger.info( `-> ${quot(check.name)} doesn't like this path` ); - continue iter_paths; + checks_pass = false; + break; } } + + if ( ! checks_pass ) continue; this.logger.info( `${hl('USING')} ${quot(entry.path)} for ${meta.pathFor}.` diff --git a/src/backend/src/codex/CodeModel.js b/src/backend/src/codex/CodeModel.js deleted file mode 100644 index d771621ca0..0000000000 --- a/src/backend/src/codex/CodeModel.js +++ /dev/null @@ -1,25 +0,0 @@ -/* - * Copyright (C) 2024 Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -class CodeModel { - static create () {} -} - -module.exports = { - CodeModel, -}; diff --git a/src/backend/src/codex/Sequence.js b/src/backend/src/codex/Sequence.js index 86e981a852..f6db2a1a92 100644 --- a/src/backend/src/codex/Sequence.js +++ b/src/backend/src/codex/Sequence.js @@ -109,7 +109,7 @@ class Sequence { async run (values) { // Initialize scope values = values || this.thisArg?.values || {}; - this.scope_.__proto__ = values; + Object.setPrototypeOf(this.scope_, values); // Run sequence for ( ; this.i < this.steps.length ; this.i++ ) { @@ -126,9 +126,8 @@ class Sequence { const parent_scope = this.scope_; this.scope_ = {}; // We could do Object.assign(this.scope_, parent_scope), but - // setting __proto__ is faster because it leverages the optimizations - // of the JS engine for the prototype chain. - this.scope_.__proto__ = parent_scope; + // setting the prototype should be faster (in theory) + Object.setPrototypeOf(this.scope_, parent_scope); if ( this.sequence_.options_.record_history ) { this.value_history_.push(this.scope_); @@ -142,6 +141,10 @@ class Sequence { this.thisArg, this, ); + if ( this.last_return_ instanceof Sequence.SequenceState ) { + this.scope_ = this.last_return_.scope_; + } + if ( this.sequence_.options_.after_each ) { await this.sequence_.options_.after_each(this, step); } diff --git a/src/backend/src/config.js b/src/backend/src/config.js index 16eebe5995..06662efb0f 100644 --- a/src/backend/src/config.js +++ b/src/backend/src/config.js @@ -31,24 +31,24 @@ config.disable_temp_users = false; config.default_user_group = '78b1b1dd-c959-44d2-b02c-8735671f9997'; config.default_temp_group = 'b7220104-7905-4985-b996-649fdcdb3c8f'; -config.max_file_size = 100_000_000_000, -config.max_thumb_size = 1_000, -config.max_fsentry_name_length = 767, +config.max_file_size = 100_000_000_000; +config.max_thumb_size = 1_000; +config.max_fsentry_name_length = 767; -config.username_regex = /^\w{1,}$/; +config.username_regex = /^\w+$/; config.username_max_length = 45; -config.subdomain_regex = /^[a-zA-Z0-9-_-]+$/; +config.subdomain_regex = /^[a-zA-Z0-9_-]+$/; config.subdomain_max_length = 60; -config.app_name_regex = /^[a-zA-Z0-9-_-]+$/; +config.app_name_regex = /^[a-zA-Z0-9_-]+$/; config.app_name_max_length = 60; config.app_title_max_length = 60; config.min_pass_length = 6; -config.strict_email_verification_required = false, -config.require_email_verification_to_publish_website = false, +config.strict_email_verification_required = false; +config.require_email_verification_to_publish_website = false; -config.kv_max_key_size = 1024, -config.kv_max_value_size = 400 * 1024, +config.kv_max_key_size = 1024; +config.kv_max_value_size = 400 * 1024; config.monitor = { metricsInterval: 60000, @@ -70,9 +70,6 @@ config.app_max_icon_size = 5*1024*1024; config.defaultjs_asset_path = '../../'; -// config.origin = config.protocol + '://' + config.domain; -// config.api_base_url = config.protocol + '://api.' + config.domain; -// config.social_card = `${config.origin}/assets/img/screenshot.png`; config.short_description = `Puter is a privacy-first personal cloud that houses all your files, apps, and games in one private and secure place, accessible from anywhere at any time.`; config.title = 'Puter'; config.company = 'Puter Technologies Inc.'; @@ -96,11 +93,12 @@ config.reserved_words = []; // set default S3 settings for this server, if any if (config.server_id) { // see if this server has a specific bucket - for (let index = 0; index < config.servers.length; index++) { - if (config.servers[index].id === config.server_id && config.servers[index].s3_bucket){ - config.s3_bucket = config.servers[index].s3_bucket; - config.s3_region = config.servers[index].region; - } + for ( const server of config.servers ) { + if ( server.id !== config.server_id ) continue; + if ( ! server.s3_bucket ) continue; + + config.s3_bucket = server.s3_bucket; + config.s3_region = server.region; } } @@ -142,15 +140,15 @@ if ( config.os.refined ) { module.exports = config; // NEW_CONFIG_LOADING +const maybe_port = config => + config.pub_port !== 80 && config.pub_port !== 443 ? ':' + config.pub_port : ''; const computed_defaults = { pub_port: config => config.http_port, - origin: config => config.protocol + '://' + config.domain + - (config.pub_port !== 80 && config.pub_port !== 443 ? ':' + config.pub_port : ''), + origin: config => config.protocol + '://' + config.domain + maybe_port(config), api_base_url: config => config.experimental_no_subdomain ? config.origin - : config.protocol + '://api.' + config.domain + - (config.pub_port !== 80 && config.pub_port !== 443 ? ':' + config.pub_port : ''), + : config.protocol + '://api.' + config.domain + maybe_port(config), social_card: config => `${config.origin}/assets/img/screenshot.png`, }; @@ -162,7 +160,7 @@ let config_to_export; // load_config() may replace const config_pointer = {}; { - config_pointer.__proto__ = config; + Object.setPrototypeOf(config_pointer, config); config_to_export = config_pointer; } @@ -173,15 +171,14 @@ const config_pointer = {}; let replacement_config = { ...o, }; - // replacement_config.__proto__ = config_pointer.__proto__; - replacement_config = deep_proto_merge(replacement_config, config_pointer.__proto__, { + replacement_config = deep_proto_merge(replacement_config, Object.getPrototypeOf(config_pointer), { preserve_flag: true, }) - config_pointer.__proto__ = replacement_config; + Object.setPrototypeOf(config_pointer, replacement_config); }; const config_api = { load_config }; - config_api.__proto__ = config_to_export; + Object.setPrototypeOf(config_api, config_to_export); config_to_export = config_api; } @@ -212,7 +209,7 @@ const config_pointer = {}; const config_runtime_values = { $: 'runtime-values' }; - config_runtime_values.__proto__ = config_to_export; + Object.setPrototypeOf(config_runtime_values, config_to_export); config_to_export = config_runtime_values // These can be difficult to find and cause painful diff --git a/src/backend/src/config/ConfigLoader.js b/src/backend/src/config/ConfigLoader.js index 0bb2bd76c8..516a13e3fc 100644 --- a/src/backend/src/config/ConfigLoader.js +++ b/src/backend/src/config/ConfigLoader.js @@ -17,7 +17,7 @@ * along with this program. If not, see . */ const { AdvancedBase } = require("@heyputer/putility"); -const { quot } = require("../util/strutil"); +const { quot } = require('@heyputer/putility').libs.string; class ConfigLoader extends AdvancedBase { static MODULES = { diff --git a/src/backend/src/data/hardcoded-permissions.js b/src/backend/src/data/hardcoded-permissions.js index abaa2940b4..0d81513ccb 100644 --- a/src/backend/src/data/hardcoded-permissions.js +++ b/src/backend/src/data/hardcoded-permissions.js @@ -84,17 +84,20 @@ const hardcoded_user_group_permissions = { 'service:hello-world:ii:hello-world': policy_perm('temp.es'), 'service:puter-kvstore:ii:puter-kvstore': policy_perm('temp.kv'), 'driver:puter-kvstore': policy_perm('temp.kv'), - 'driver:puter-notifications': policy_perm('temp.es'), - 'driver:puter-apps': policy_perm('temp.es'), - 'driver:puter-subdomains': policy_perm('temp.es'), + 'service:puter-notifications:ii:crud-q': policy_perm('temp.es'), + 'service:puter-apps:ii:crud-q': policy_perm('temp.es'), + 'service:puter-subdomains:ii:crud-q': policy_perm('temp.es'), + 'service:es\\Cnotification:ii:crud-q': policy_perm('user.es'), + 'service:es\\Capp:ii:crud-q': policy_perm('user.es'), + 'service:es\\Csubdomain:ii:crud-q': policy_perm('user.es'), }, '78b1b1dd-c959-44d2-b02c-8735671f9997': { 'service:hello-world:ii:hello-world': policy_perm('user.es'), 'service:puter-kvstore:ii:puter-kvstore': policy_perm('user.kv'), 'driver:puter-kvstore': policy_perm('user.kv'), - 'driver:puter-notifications': policy_perm('user.es'), - 'driver:puter-apps': policy_perm('user.es'), - 'driver:puter-subdomains': policy_perm('user.es'), + 'service:es\\Cnotification:ii:crud-q': policy_perm('user.es'), + 'service:es\\Capp:ii:crud-q': policy_perm('user.es'), + 'service:es\\Csubdomain:ii:crud-q': policy_perm('user.es'), }, }, }; diff --git a/src/backend/src/definitions/Driver.js b/src/backend/src/definitions/Driver.js deleted file mode 100644 index 4bf79af524..0000000000 --- a/src/backend/src/definitions/Driver.js +++ /dev/null @@ -1,192 +0,0 @@ -/* - * Copyright (C) 2024 Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { AdvancedBase } = require("@heyputer/putility"); -const { Context } = require('../util/context') -const APIError = require("../api/APIError"); -const { AppUnderUserActorType, UserActorType } = require("../services/auth/Actor"); -const { BaseOperation } = require("../services/OperationTraceService"); -const { CodeUtil } = require("../codex/CodeUtil"); - -/** - * Base class for all driver implementations. - * - * @deprecated - we use traits on services now. This class is kept for compatibility - * with EntityStoreImplementation and DBKVStore which still use this. - */ -class Driver extends AdvancedBase { - constructor (...a) { - super(...a); - const methods = this._get_merged_static_object('METHODS'); - // Turn each method into an operation - for ( const k in methods ) { - methods[k] = CodeUtil.mrwrap(methods[k], BaseOperation, { - name: `${this.constructor.ID}:${k}`, - }); - }; - this.methods = methods; - this.sla = this._get_merged_static_object('SLA'); - } - - async call (method, args) { - if ( ! this.methods[method] ) { - throw new Error(`method not found: ${method}`); - } - - const pseudo_this = Object.assign({}, this); - - const context = Context.get(); - pseudo_this.context = context; - pseudo_this.services = context.get('services'); - const services = context.get('services'); - pseudo_this.log = services.get('log-service').create(this.constructor.name); - - await this._sla_enforcement(method); - - return await this.methods[method].call(pseudo_this, args); - } - - async _sla_enforcement (method) { - const context = Context.get(); - const services = context.get('services'); - const method_key = `${this.constructor.ID}:${method}`; - const svc_sla = services.get('sla'); - - // System SLA enforcement - { - const sla_key = `driver:impl:${method_key}`; - const sla = await svc_sla.get('system', sla_key); - - const sys_method_key = `system:${method_key}`; - - // short-term rate limiting - if ( sla?.rate_limit ) { - const svc_rateLimit = services.get('rate-limit'); - let eventual_success = false; - for ( let i = 0 ; i < 60 ; i++ ) { - try { - await svc_rateLimit.check_and_increment(sys_method_key, sla.rate_limit.max, sla.rate_limit.period); - eventual_success = true; - break; - } catch ( e ) { - if ( - ! ( e instanceof APIError ) || - e.fields.code !== 'rate_limit_exceeded' - ) throw e; - await new Promise((resolve) => setTimeout(resolve, 1000)); - } - } - if ( ! eventual_success ) { - throw APIError.create('server_rate_exceeded'); - } - } - } - - // test_mode is checked to prevent rate limiting when it is enabled - const test_mode = context.get('test_mode'); - - // User SLA enforcement - { - const actor = context.get('actor').get_related_actor(UserActorType); - - const user_is_verified = !! actor.type.user.email_confirmed; - - const sla_key = `driver:impl:${method_key}`; - const sla = await svc_sla.get( - user_is_verified ? 'user_verified' : 'user_unverified', - sla_key - ); - - const user_method_key = `actor:${actor.uid}:${method_key}`; - - // short-term rate limiting - if ( sla?.rate_limit ) { - const svc_rateLimit = services.get('rate-limit'); - await svc_rateLimit.check_and_increment(method_key, sla.rate_limit.max, sla.rate_limit.period); - } - - // long-term rate limiting - if ( sla?.monthly_limit && ! test_mode ) { - const svc_monthlyUsage = services.get('monthly-usage'); - const count = await svc_monthlyUsage.check( - actor, { - 'driver.interface': this.constructor.INTERFACE, - 'driver.implementation': this.constructor.ID, - 'driver.method': method, - }); - if ( count >= sla.monthly_limit ) { - throw APIError.create('monthly_limit_exceeded', null, { - method_key, - limit: sla.monthly_limit, - }); - } - } - } - - // App SLA enforcement - await (async () => { - const actor = context.get('actor'); - if ( ! ( actor.type instanceof AppUnderUserActorType ) ) return; - - const sla_key = `driver:impl:${method_key}`; - const sla = await svc_sla.get('app_default', sla_key); - - // long-term rate limiting - if ( sla?.monthly_limit && ! test_mode ) { - const svc_monthlyUsage = services.get('monthly-usage'); - const count = await svc_monthlyUsage.check( - actor, { - 'driver.interface': this.constructor.INTERFACE, - 'driver.implementation': this.constructor.ID, - 'driver.method': method, - }); - if ( count >= sla.monthly_limit ) { - throw APIError.create('monthly_limit_exceeded', null, { - method_key, - limit: sla.monthly_limit, - }); - } - } - })(); - - // Record monthly usage - if ( ! test_mode ) { - const actor = context.get('actor'); - const svc_monthlyUsage = services.get('monthly-usage'); - const extra = { - 'driver.interface': this.constructor.INTERFACE, - 'driver.implementation': this.constructor.ID, - 'driver.method': method, - ...(this.get_usage_extra ? this.get_usage_extra() : {}), - }; - await svc_monthlyUsage.increment(actor, method_key, extra); - } - } - - async get_response_meta () { - return { - driver: this.constructor.ID, - driver_version: this.constructor.VERSION, - driver_interface: this.constructor.INTERFACE, - }; - } -} - -module.exports = { - Driver, -}; diff --git a/src/backend/src/drivers/DBKVStore.js b/src/backend/src/drivers/DBKVStore.js deleted file mode 100644 index 59b994bb6c..0000000000 --- a/src/backend/src/drivers/DBKVStore.js +++ /dev/null @@ -1,217 +0,0 @@ -/* - * Copyright (C) 2024 Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const config = require("../config"); -const APIError = require("../api/APIError"); -const { DB_READ, DB_WRITE } = require("../services/database/consts"); -const { Driver } = require("../definitions/Driver"); -const { get_app } = require("../helpers"); - -class DBKVStore extends Driver { - static ID = 'public-db-kvstore'; - static VERSION = '0.0.0'; - static INTERFACE = 'puter-kvstore'; - static MODULES = { - murmurhash: require('murmurhash'), - } - static METHODS = { - get: async function ({ app_uid, key }) { - const actor = this.context.get('actor'); - - // If the actor is an app then it gets its own KV store. - // The way this is implemented isn't ideal for future behaviour; - // a KV implementation specified by the user would have parameters - // that are scoped to the app, so this should eventually be - // changed to get the app ID from the same interface that would - // be used to obtain per-app user-specified implementation params. - let app = actor.type?.app ?? undefined; - const user = actor.type?.user ?? undefined; - - if ( ! user ) throw new Error('User not found'); - - if ( ! app && app_uid ) { - app = await get_app({ uid: app_uid }); - } - - const db = this.services.get('database').get(DB_READ, 'kvstore'); - const key_hash = this.modules.murmurhash.v3(key); - const kv = app ? await db.read( - `SELECT * FROM kv WHERE user_id=? AND app=? AND kkey_hash=? LIMIT 1`, - [ user.id, app.uid, key_hash ] - ) : await db.read( - `SELECT * FROM kv WHERE user_id=? AND (app IS NULL OR app = 'global') AND kkey_hash=? LIMIT 1`, - [ user.id, key_hash ] - ); - - if ( kv[0] ) kv[0].value = db.case({ - mysql: () => kv[0].value, - otherwise: () => JSON.parse(kv[0].value ?? 'null'), - })(); - - return kv[0]?.value ?? null; - }, - set: async function ({ app_uid, key, value }) { - const actor = this.context.get('actor'); - - // Validate the key - // get() doesn't String() the key but it only passes it to - // murmurhash.v3() so it doesn't need to ¯\_(ツ)_/¯ - key = String(key); - if ( Buffer.byteLength(key, 'utf8') > config.kv_max_key_size ) { - throw new Error(`key is too large. Max size is ${config.kv_max_key_size}.`); - } - - // Validate the value - value = value === undefined ? null : value; - if ( - value !== null && - Buffer.byteLength(JSON.stringify(value), 'utf8') > - config.kv_max_value_size - ) { - throw new Error(`value is too large. Max size is ${config.kv_max_value_size}.`); - } - - let app = actor.type?.app ?? undefined; - const user = actor.type?.user ?? undefined; - if ( ! user ) throw new Error('User not found'); - - if ( ! app && app_uid ) { - app = await get_app({ uid: app_uid }); - } - - const db = this.services.get('database').get(DB_WRITE, 'kvstore'); - const key_hash = this.modules.murmurhash.v3(key); - - try { - await db.write( - `INSERT INTO kv (user_id, app, kkey_hash, kkey, value) - VALUES (?, ?, ?, ?, ?) ` + - db.case({ - mysql: 'ON DUPLICATE KEY UPDATE value = ?', - sqlite: 'ON CONFLICT(user_id, app, kkey_hash) DO UPDATE SET value = excluded.value', - }), - [ - user.id, app?.uid ?? 'global', key_hash, key, - JSON.stringify(value), - ...db.case({ mysql: [value], otherwise: [] }), - ] - ); - } catch (e) { - // I discovered that my .sqlite file was corrupted and the update - // above didn't work. The current database initialization does not - // cause this issue so I'm adding this log as a safeguard. - // - KernelDeimos / ED - const svc_error = this.services.get('error-service'); - svc_error.report('kvstore:sqlite_error', { - message: 'Broken database version - please contact maintainers', - source: e, - }); - } - - return true; - }, - del: async function ({ app_uid, key }) { - const actor = this.context.get('actor'); - - let app = actor.type?.app ?? undefined; - const user = actor.type?.user ?? undefined; - if ( ! user ) throw new Error('User not found'); - - if ( ! app && app_uid ) { - app = await get_app({ uid: app_uid }); - } - - const db = this.services.get('database').get(DB_WRITE, 'kvstore'); - const key_hash = this.modules.murmurhash.v3(key); - - await db.write( - `DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?`, - [ user.id, app?.uid ?? 'global', key_hash ] - ); - - return true; - }, - list: async function ({ app_uid, as }) { - const actor = this.context.get('actor'); - - let app = actor.type?.app ?? undefined; - const user = actor.type?.user ?? undefined; - - if ( ! app && app_uid ) { - app = await get_app({ uid: app_uid }); - } - - if ( ! user ) throw new Error('User not found'); - - const db = this.services.get('database').get(DB_READ, 'kvstore'); - let rows = app ? await db.read( - `SELECT kkey, value FROM kv WHERE user_id=? AND app=?`, - [ user.id, app.uid ] - ) : await db.read( - `SELECT kkey, value FROM kv WHERE user_id=? AND (app IS NULL OR app = 'global')`, - [ user.id ] - ); - - rows = rows.map(row => ({ - key: row.kkey, - value: db.case({ - mysql: () => row.value, - otherwise: () => JSON.parse(row.value ?? 'null') - })(), - })); - - as = as || 'entries'; - - if ( ! ['keys','values','entries'].includes(as) ) { - throw APIError.create('field_invalid', null, { - key: 'as', - expected: '"keys", "values", or "entries"', - }); - } - - if ( as === 'keys' ) rows = rows.map(row => row.key); - else if ( as === 'values' ) rows = rows.map(row => row.value); - - return rows; - }, - flush: async function ({ app_uid }) { - const actor = this.context.get('actor'); - - let app = actor.type?.app ?? undefined; - const user = actor.type?.user ?? undefined; - if ( ! user ) throw new Error('User not found'); - - if ( ! app && app_uid ) { - app = await get_app({ uid: app_uid }); - } - - const db = this.services.get('database').get(DB_WRITE, 'kvstore'); - - await db.write( - `DELETE FROM kv WHERE user_id=? AND app=?`, - [ user.id, app?.uid ?? 'global' ] - ); - - return true; - } - } -} - -module.exports = { - DBKVStore, -} diff --git a/src/backend/src/drivers/EntityStoreImplementation.js b/src/backend/src/drivers/EntityStoreImplementation.js deleted file mode 100644 index a84ae8bd37..0000000000 --- a/src/backend/src/drivers/EntityStoreImplementation.js +++ /dev/null @@ -1,171 +0,0 @@ -/* - * Copyright (C) 2024 Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const APIError = require("../api/APIError"); -const { Driver } = require("../definitions/Driver"); -const { Entity } = require("../om/entitystorage/Entity"); -const { Or, And, Eq } = require("../om/query/query"); - -const _fetch_based_on_complex_id = async (self, id) => { - // Ensure `id` is an object and get its keys - if ( ! id || typeof id !== 'object' || Array.isArray(id) ) { - throw APIError.create('invalid_id', null, { id }); - } - - const id_keys = Object.keys(id); - // sort keys alphabetically - id_keys.sort(); - - // Ensure key set is valid based on redundant keys listing - const svc_es = self.services.get(self.service); - const redundant_identifiers = svc_es.om.redundant_identifiers ?? []; - - let match_found = false; - for ( let key of redundant_identifiers ) { - // Either a single key or a list - key = Array.isArray(key) ? key : [key]; - - // All keys in the list must be present in the id - for ( let i=0 ; i < key.length ; i++ ) { - if ( ! id_keys.includes(key[i]) ) { - break; - } - if ( i === key.length - 1 ) { - match_found = true; - break; - } - } - } - - if ( ! match_found ) { - throw APIError.create('invalid_id', null, { id }); - } - - // Construct a query predicate based on the keys - const key_eqs = []; - for ( const key of id_keys ) { - key_eqs.push(new Eq({ - key, - value: id[key], - })); - } - let predicate = new And({ children: key_eqs }); - - // Perform a select - const entity = await svc_es.read({ predicate }); - if ( ! entity ) { - return null; - } - - // Ensure there is only one result - return entity; -} - -const _fetch_based_on_either_id = async (self, uid, id) => { - if ( uid ) { - const svc_es = self.services.get(self.service); - return await svc_es.read(uid); - } - - return await _fetch_based_on_complex_id(self, id); -} - -class EntityStoreImplementation extends Driver { - constructor ({ service }) { - super(); - this.service = service; - } - get_usage_extra () { - return { - ['driver.interface']: 'puter-es', - ['driver.implementation']: 'puter-es:' + this.service, - }; - } - static METHODS = { - create: async function ({ object, options }) { - const svc_es = this.services.get(this.service); - if ( object.hasOwnProperty(svc_es.om.primary_identifier) ) { - throw APIError.create('field_not_allowed_for_create', null, { key: svc_es.om.primary_identifier }); - } - const entity = await Entity.create({ om: svc_es.om }, object); - return await svc_es.create(entity, options); - }, - update: async function ({ object, id, options }) { - const svc_es = this.services.get(this.service); - // if ( ! object.hasOwnProperty(svc_es.om.primary_identifier) ) { - // throw APIError.create('field_required_for_update', null, { key: svc_es.om.primary_identifier }); - // } - const entity = await Entity.create({ om: svc_es.om }, object); - return await svc_es.update(entity, id, options); - }, - upsert: async function ({ object, id, options }) { - const svc_es = this.services.get(this.service); - const entity = await Entity.create({ om: svc_es.om }, object); - return await svc_es.upsert(entity, id, options); - }, - read: async function ({ uid, id }) { - if ( ! uid && ! id ) { - throw APIError.create('xor_field_missing', null, { - names: ['uid', 'id'], - }); - } - - const entity = await _fetch_based_on_either_id(this, uid, id); - if ( ! entity ) { - throw APIError.create('entity_not_found', null, { - identifier: uid - }); - } - return await entity.get_client_safe(); - }, - select: async function (options) { - const svc_es = this.services.get(this.service); - const entities = await svc_es.select(options); - const client_safe_entities = []; - for ( const entity of entities ) { - client_safe_entities.push(await entity.get_client_safe()); - } - return client_safe_entities; - }, - delete: async function ({ uid, id }) { - if ( ! uid && ! id ) { - throw APIError.create('xor_field_missing', null, { - names: ['uid', 'id'], - }); - } - - if ( id && ! uid ) { - const entity = await _fetch_based_on_complex_id(this, id); - if ( ! entity ) { - throw APIError.create('entity_not_found', null, { - identifier: id - }); - } - const svc_es = this.services.get(this.service); - uid = await entity.get(svc_es.om.primary_identifier); - } - - const svc_es = this.services.get(this.service); - return await svc_es.delete(uid); - }, - }; -} - -module.exports = { - EntityStoreImplementation, -}; diff --git a/src/backend/src/drivers/HelloWorld.js b/src/backend/src/drivers/HelloWorld.js deleted file mode 100644 index 058e5d126f..0000000000 --- a/src/backend/src/drivers/HelloWorld.js +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (C) 2024 Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { Driver } = require("../definitions/Driver"); - -class HelloWorld extends Driver { - static ID = 'public-helloworld'; - static VERSION = '0.0.0'; - static INTERFACE = 'helloworld'; - static SLA = { - greet: { - rate_limit: { - max: 10, - period: 30000, - }, - monthly_limit: Math.pow(1, 6), - }, - } - static METHODS = { - greet: async function ({ subject }) { - return `Hello, ${subject ?? 'World'}!` - } - } -} - -module.exports = { - HelloWorld, -}; diff --git a/src/backend/src/errors/error_help_details.js b/src/backend/src/errors/error_help_details.js index 0a37b4861d..8511012a06 100644 --- a/src/backend/src/errors/error_help_details.js +++ b/src/backend/src/errors/error_help_details.js @@ -16,7 +16,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -const { quot, osclink } = require("../util/strutil"); +const { quot, osclink } = require('@heyputer/putility').libs.string; const reused = { runtime_env_references: [ @@ -50,7 +50,7 @@ const error_help_details = [ apply (more) { more.references = [ ...reused.runtime_env_references, - ] + ]; } }, { @@ -68,10 +68,10 @@ const error_help_details = [ { title: 'Set CONFIG_PATH or RUNTIME_PATH environment variable', }, - ], + ]; more.references = [ ...reused.runtime_env_references, - ] + ]; } }, { @@ -83,7 +83,7 @@ const error_help_details = [ { title: 'Create a valid config file', }, - ] + ]; } }, { @@ -112,7 +112,7 @@ const error_help_details = [ use: 'describes why this error occurs', url: 'https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Errors/Invalid_const_assignment' }, - ] + ]; } }, { @@ -122,7 +122,7 @@ const error_help_details = [ apply (more) { more.notes = [ 'It looks like this might be our fault.', - ] + ]; more.solutions = [ { title: `Check for an issue on ` + @@ -135,7 +135,7 @@ const error_help_details = [ 'create one' ) + '.' } - ] + ]; } }, { diff --git a/src/backend/src/filesystem/FSNodeContext.js b/src/backend/src/filesystem/FSNodeContext.js index d575206601..e86b2a9031 100644 --- a/src/backend/src/filesystem/FSNodeContext.js +++ b/src/backend/src/filesystem/FSNodeContext.js @@ -18,11 +18,13 @@ */ const { get_user, get_dir_size, id2path, id2uuid, is_empty, is_shared_with_anyone, suggest_app_for_fsentry, get_app } = require("../helpers"); +const putility = require('@heyputer/putility'); +const { MultiDetachable } = putility.libs.listener; +const { TDetachable } = putility.traits; const config = require("../config"); const _path = require('path'); const { NodeInternalIDSelector, NodeChildSelector, NodeUIDSelector, RootNodeSelector, NodePathSelector } = require("./node/selectors"); const { Context } = require("../util/context"); -const { MultiDetachable } = require("../util/listenerutil"); const { NodeRawEntrySelector } = require("./node/selectors"); const { DB_READ } = require("../services/database/consts"); const { UserActorType } = require("../services/auth/Actor"); @@ -87,7 +89,7 @@ module.exports = class FSNodeContext { this.fs = fs; // Decorate all fetch methods with otel span - // TODO: language tool for traits; this is a trait + // TODO: Apply method decorators using a putility class feature const fetch_methods = [ 'fetchEntry', 'fetchPath', @@ -271,8 +273,6 @@ module.exports = class FSNodeContext { resourceService, } = Context.get('services').values; - // await this.fs.resourceService - // .waitForResource(this.selector); if ( fetch_entry_options.tracer == null ) { fetch_entry_options.tracer = traceService.tracer; } @@ -288,12 +288,8 @@ module.exports = class FSNodeContext { await new Promise (rslv => { const detachables = new MultiDetachable(); - let resolved = false; - const callback = (resolver) => { - // NOTE: commented out for now because it's too verbose - resolved = true; - detachables.detach(); + detachables.as(TDetachable).detach(); rslv(); } @@ -499,8 +495,7 @@ module.exports = class FSNodeContext { [this.entry.id] ); const versions_tidy = []; - for (let index = 0; index < versions.length; index++) { - const version = versions[index]; + for ( const version of versions ) { let username = version.user_id ? (await get_user({id: version.user_id})).username : null; versions_tidy.push({ id: version.version_id, @@ -518,13 +513,15 @@ module.exports = class FSNodeContext { /** * Fetches the size of a file or directory if it was not * already fetched. - * @param {object} user the user is needed to fetch the size */ - async fetchSize (user) { + async fetchSize () { const { fsEntryService } = Context.get('services').values; // we already have the size for files - if ( ! this.entry.is_dir ) return; + if ( ! this.entry.is_dir ) { + await this.fetchEntry(); + return this.entry.size; + } this.entry.size = await fsEntryService.get_recursive_size( this.entry.uuid, @@ -551,14 +548,6 @@ module.exports = class FSNodeContext { this.entry.is_empty = await is_empty(this.uid); } - // TODO: this is currently not called anywhere; for now it - // will never be fetched since sharing is not a priority. - async fetchIsShared () { - if ( ! this.mysql_id ) return; - - this.entry.is_shared = await is_shared_with_anyone(this.mysql_id); - } - async fetchAll(fsEntryFetcher, user, force) { await this.fetchEntry({ thumbnail: true }); await this.fetchSubdomains(user); @@ -610,7 +599,6 @@ module.exports = class FSNodeContext { ); } if ( ! this.path ) { - // console.log('PATH WAS NOT ON ENTRY', this); await this.fetchPath(); } if ( ! this.path ) { diff --git a/src/backend/src/filesystem/FSOperationContext.js b/src/backend/src/filesystem/FSOperationContext.js deleted file mode 100644 index 53fa0b302c..0000000000 --- a/src/backend/src/filesystem/FSOperationContext.js +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright (C) 2024 Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const PerformanceMonitor = require('../monitor/PerformanceMonitor'); - -const FSNodeContext = require('./FSNodeContext'); -const FSAccessContext = require('./FSAccessContext'); -const { Context } = require('../util/context'); - -/** - * FSOperationContext represents a single operation on the filesystem. - * - * FSOperationContext is used to record events such as side-effects - * which occur during a high-level filesystem operation. It is also - * responsible for generating a client-safe result which describes - * the operation. - */ -module.exports = class FSOperationContext { - // TODO: rename this.fs to this.access - constructor (op_name, context, options) { - // migration: fs:create-service - // TODO: rename this.fs to this.access - // NOTE: the 2nd parameter of this constructor - // was called `fs` and was expected to be FSAccessContext. - // Now it should be a context object holding the services - // container. context.access is the FSAccessContext. - if ( context instanceof FSAccessContext ) { - this.fs = context; - } else if ( context ) { - this.context = context; - this.fs = context.access; - } else { - const x = Context.get(); - this.fs = {}; - this.fs.traceService = x.get('services').get('traceService'); - } - - this.name = op_name; - this.events = []; - this.parent_dirs_created = []; - this.created = []; - this.fields = {}; - this.safeFields = {}; - - this.valueListeners_ = {}; - this.valueFactories_ = {}; - this.values_ = {}; - this.rejections_ = {}; - - this.tasks_ = []; - - this.currentCheckpoint_ = 'checkpoint not set'; - - if ( options.parent_operation ) { - this.parent = options.parent_operation; - } - - this.donePromise = new Promise((resolve, reject) => { - this.doneResolve = resolve; - this.doneReject = reject; - }); - - // migration: arch:trace-service:move-outta-fs - if ( this.fs.traceService ) { - // Set 'span_' to current active span - const { context, trace } = require('@opentelemetry/api'); - this.span_ = trace.getSpan(context.active()); - } - - this.monitor = PerformanceMonitor.createContext(`fs.${op_name}`); - } - - checkpoint (label) { - this.currentCheckpoint_ = label; - } - - async addTask (name, fn) { - const task = { - name, - operations: [], - promise: Promise.resolve(), - }; - - const taskContext = { - registerOperation: op => { - task.operations.push(op); - task.promise = task.promise.then(() => op.awaitDone()); - } - }; - - const monitor = PerformanceMonitor.createContext('fs.rm'); - monitor.label(`task:${name}`); - task.promise = task.promise.then(() => fn(taskContext)); - this.tasks_.push(task); - - let last_promise = null; - while ( task.promise !== last_promise ) { - last_promise = task.promise; - await task.promise; - } - // await task.promise; - - monitor.stamp(); - monitor.end(); - } - - get span () { return this.span_; } - - recordParentDirCreated (fsNode) { - if ( ! fsNode ) { - throw new Error( - 'falsy value to recordParentDirCreated', - fsNode, - ); - } - this.parent_dirs_created.push(fsNode); - } - - recordCreated (fsNode) { - this.created.push(fsNode); - } - - set (field, value) { - this.fields[field] = value; - } - - async set_now (field, value) { - this.fields[field] = value; - if ( value instanceof FSNodeContext ) { - this.safeFields[field] = await value.getSafeEntry(); - } - } - - get (field) { - return this.fields[field]; - } - - complete (options) { - options = options ?? {}; - - if ( this.parent ) { - for ( const fsNode of this.parent_dirs_created ) { - this.parent.recordParentDirCreated(fsNode); - } - - for ( const fsNode of this.created ) { - this.parent.recordCreated(fsNode); - } - } - - if ( this.tasks_.length > 0 ) { - // TODO: it's mutating input options, which is not ideal - if ( ! options.after ) options.after = []; - - options.after.push( - this.tasks_.map(task => task.promise) - ); - } - - if ( options.after ) { - const thingsToWaitFor = options.after.map(item => { - if ( item.awaitDone ) return item.awaitDone; - return item; - }); - (async () => { - await Promise.all(thingsToWaitFor); - this.doneResolve(); - })(); - return; - } - - this.doneResolve(); - } - - onComplete(fn) { - this.donePromise.then(fn); - } - - awaitDone () { - return this.donePromise; - } - - provideValue (key, value) { - this.values_[key] = value; - - let listeners = this.valueListeners_[key]; - if ( ! listeners ) return; - - delete this.valueListeners_[key]; - - for ( let listener of listeners ) { - if ( Array.isArray(listener) ) listener = listener[0]; - listener(value); - } - } - - rejectValue (key, err) { - this.rejections_[key] = err; - - let listeners = this.valueListeners_[key]; - if ( ! listeners ) return; - - delete this.valueListeners_[key]; - - for ( let listener of listeners ) { - if ( ! Array.isArray(listener) ) continue; - if ( ! listener[1] ) continue; - listener = listener[1]; - - listener(err); - } - } - - awaitValue (key) { - return new Promise ((rslv, rjct) => { - this.onValue(key, rslv, rjct); - }); - } - - onValue (key, fn, rjct) { - if ( this.values_[key] ) { - fn(this.values_[key]); - return; - } - - if ( this.rejections_[key] ) { - if ( rjct ) { - rjct(this.rejections_[key]); - } else throw this.rejections_[key]; - return; - } - - if ( ! this.valueListeners_[key] ) { - this.valueListeners_[key] = []; - } - this.valueListeners_[key].push([fn, rjct]); - - if ( this.valueFactories_[key] ) { - const fn = this.valueFactories_[key]; - delete this.valueFactories_[key]; - (async () => { - try { - const value = await fn(); - this.provideValue(key, value); - } catch (e) { - this.rejectValue(key, e); - } - })(); - } - } - - async setFactory (key, factoryFn) { - if ( this.valueListeners_[key] ) { - let v; - try { - v = await factoryFn(); - } catch (e) { - this.rejectValue(key, e); - } - this.provideValue(key, v); - return; - } - - this.valueFactories_[key] = factoryFn; - } - - /** - * Listen for another operation to complete, and then - * complete this operation. This is useful for operations - * which delegate to other operations. - * - * @param {FSOperationContext} other - * @returns {FSOperationContext} this - */ - completedBy (other) { - other.onComplete(() => { - this.complete(); - }); - - return this; - } - - /** - * Produces an object which describes the operation in a - * way that is intended to be sent to the client. - * - * @returns {Promise} - */ - async getClientSafeResult () { - const result = {}; - for ( const field in this.fields ) { - if ( this.fields[field] instanceof FSNodeContext ) { - result[field] = this.safeFields[field] ?? - await this.fields[field].getSafeEntry(); - continue; - } - - result[field] = this.fields[field]; - } - - result.parent_dirs_created = []; - for ( const fsNode of this.parent_dirs_created ) { - const fsNodeResult = await fsNode.getSafeEntry(); - result.parent_dirs_created.push(fsNodeResult); - } - - return result; - } -} diff --git a/src/backend/src/filesystem/FilesystemService.js b/src/backend/src/filesystem/FilesystemService.js index 48da46c71d..035481463a 100644 --- a/src/backend/src/filesystem/FilesystemService.js +++ b/src/backend/src/filesystem/FilesystemService.js @@ -17,13 +17,9 @@ * along with this program. If not, see . */ // TODO: database access can be a service -const { ResourceService, RESOURCE_STATUS_PENDING_CREATE } = require('./storage/ResourceService'); -const DatabaseFSEntryFetcher = require("./storage/DatabaseFSEntryFetcher"); -const { DatabaseFSEntryService } = require('./storage/DatabaseFSEntryService'); -const { SizeService } = require('./storage/SizeService'); +const { RESOURCE_STATUS_PENDING_CREATE } = require('../modules/puterfs/ResourceService.js'); const { TraceService } = require('../services/TraceService.js'); const FSAccessContext = require('./FSAccessContext.js'); -const SystemFSEntryService = require('./storage/SystemFSEntryService.js'); const PerformanceMonitor = require('../monitor/PerformanceMonitor.js'); const { NodePathSelector, NodeUIDSelector, NodeInternalIDSelector } = require('./node/selectors.js'); const FSNodeContext = require('./FSNodeContext.js'); @@ -44,29 +40,15 @@ class FilesystemService extends BaseService { static MODULES = { _path: require('path'), uuidv4: require('uuid').v4, - socketio: require('../socketio.js'), config: require('../config.js'), } old_constructor (args) { - // super(args); const { services } = args; - // this.services = services; - - services.registerService('resourceService', ResourceService); - services.registerService('sizeService', SizeService); services.registerService('traceService', TraceService); - // TODO: [fs:remove-separate-updater-and-fetcher] - services.set('fsEntryFetcher', new DatabaseFSEntryFetcher({ - services: services, - })); - services.registerService('fsEntryService', DatabaseFSEntryService); - // The new fs entry service - services.registerService('systemFSEntryService', SystemFSEntryService); - this.log = services.get('log-service').create('filesystem-service'); // used by update_child_paths @@ -81,27 +63,6 @@ class FilesystemService extends BaseService { .obtain('fs.fsentry:path') .exec(entry.uuid); }); - - - // Decorate methods with otel span - // TODO: language tool for traits; this is a trait - const span_methods = [ - 'write', 'mkdir', 'rm', 'mv', 'cp', 'read', 'stat', - 'mkdir_2', - 'update_child_paths', - ]; - for ( const method of span_methods ) { - const original_method = this[method]; - this[method] = async (...args) => { - const tracer = services.get('traceService').tracer; - let result; - await tracer.startActiveSpan(`fs-svc:${method}`, async span => { - result = await original_method.call(this, ...args); - span.end(); - }); - return result; - } - } } async _init () { @@ -261,8 +222,7 @@ class FilesystemService extends BaseService { await target.fetchEntry({ thumbnail: true }); const { _path, uuidv4 } = this.modules; - const resourceService = this.services.get('resourceService'); - const systemFSEntryService = this.services.get('systemFSEntryService'); + const svc_fsEntry = this.services.get('fsEntryService'); const ts = Math.round(Date.now() / 1000); const uid = uuidv4(); @@ -292,7 +252,7 @@ class FilesystemService extends BaseService { this.log.debug('creating fsentry', { fsentry: raw_fsentry }) - const entryOp = await systemFSEntryService.insert(raw_fsentry); + const entryOp = await svc_fsEntry.insert(raw_fsentry); console.log('entry op', entryOp); @@ -329,7 +289,7 @@ class FilesystemService extends BaseService { const { _path, uuidv4 } = this.modules; const resourceService = this.services.get('resourceService'); - const systemFSEntryService = this.services.get('systemFSEntryService'); + const svc_fsEntry = this.services.get('fsEntryService'); const ts = Math.round(Date.now() / 1000); const uid = uuidv4(); @@ -356,7 +316,7 @@ class FilesystemService extends BaseService { this.log.debug('creating symlink', { fsentry: raw_fsentry }) - const entryOp = await systemFSEntryService.insert(raw_fsentry); + const entryOp = await svc_fsEntry.insert(raw_fsentry); (async () => { await entryOp.awaitDone(); diff --git a/src/backend/src/filesystem/backends/Test.js b/src/backend/src/filesystem/backends/Test.js deleted file mode 100644 index d2f35d624b..0000000000 --- a/src/backend/src/filesystem/backends/Test.js +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright (C) 2024 Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.Test = void 0; -class Test { -} -exports.Test = Test; diff --git a/src/backend/src/filesystem/backends/Test.ts b/src/backend/src/filesystem/backends/Test.ts deleted file mode 100644 index 6b8585ca9c..0000000000 --- a/src/backend/src/filesystem/backends/Test.ts +++ /dev/null @@ -1,3 +0,0 @@ -export class Test { - // -} \ No newline at end of file diff --git a/src/backend/src/filesystem/batch/BatchExecutor.js b/src/backend/src/filesystem/batch/BatchExecutor.js index a0ed2ea5ea..db452316a7 100644 --- a/src/backend/src/filesystem/batch/BatchExecutor.js +++ b/src/backend/src/filesystem/batch/BatchExecutor.js @@ -19,11 +19,11 @@ const { AdvancedBase } = require('@heyputer/putility'); const PathResolver = require('../../routers/filesystem_api/batch/PathResolver'); const commands = require('./commands').commands; -const { WorkUnit } = require('../../services/runtime-analysis/ExpectationService'); const APIError = require('../../api/APIError'); const { Context } = require('../../util/context'); const config = require('../../config'); -const { TeePromise } = require('../../util/promise'); +const { TeePromise } = require('@heyputer/putility').libs.promise; +const { WorkUnit } = require('../../modules/core/lib/expect'); class BatchExecutor extends AdvancedBase { constructor (x, { actor, log, errors }) { diff --git a/src/backend/src/filesystem/core/.gitignore b/src/backend/src/filesystem/core/.gitignore deleted file mode 100644 index dc7e0c2bf3..0000000000 --- a/src/backend/src/filesystem/core/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -# Typescript directory -*.js \ No newline at end of file diff --git a/src/backend/src/filesystem/core/BackendAPI.ts b/src/backend/src/filesystem/core/BackendAPI.ts deleted file mode 100644 index 5eb1455352..0000000000 --- a/src/backend/src/filesystem/core/BackendAPI.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { ISelector } from "./Selector"; - -type PuterUserID = number; - -export const enum FSBackendSupportFlags { - None = 0, - - // Platform-related flags - PlatformCaseSensitive = 1 << 1, - - // Puter support flags - // PuterStatOwner indicates the backend can store `user_id` - PuterStatOwner = 1 << 2, - // PuterStatApp indicates the backend can store `associated_app_id` - PuterStatApp = 1 << 3, - - // DetailVerboseReaddir indicates the backend will provide a full - // stat() result for each entry in readdir(). - DetailVerboseReaddir = 1 << 4, -} - -export const enum FSNodeType { - File, - Directory, - PuterShortcut, - SymbolicLink, - KVStore, - Socket, -} - -export interface IOverwriteOptions { - readonly overwrite: boolean; - UserID: PuterUserID, -} - -export interface IWriteOptions extends IOverwriteOptions { - readonly create: boolean; -} - -export interface IDeleteOptions { - readonly recursive: boolean; -} - -export interface IStatOptions { - followSymlinks?: boolean; -} - -export interface IStatResult { - uuid: string; - name: string; - type: FSNodeType; - size: number; - mtime: Date; - ctime: Date; - atime: Date; - immutable: boolean; -} - -export interface IMiniStatResult { - uuid: string; - name: string; - type: FSNodeType; -} - -type ReaddirResult = IMiniStatResult | IStatResult; - -export interface IMkdirOptions { - // Not for permission checks by the storage backend. - // A supporting storage backend will simply store this and - // return it in the stat() call. - UserID: PuterUserID, -} - -export interface BackendAPI { - stat (selector: ISelector, options: IStatOptions): Promise; - readdir (selector: ISelector): Promise<[string, ReaddirResult][]>; - - mkdir (selector: ISelector, name: string): Promise; - copy (from: ISelector, to: ISelector, options: IOverwriteOptions): Promise; - rename (from: ISelector, to: ISelector, options: IOverwriteOptions): Promise; - delete (selector: ISelector, options: IDeleteOptions): Promise; - - read_file (selector: ISelector): Promise; - write_file (selector: ISelector, data: Buffer, options: IOverwriteOptions): Promise; -} diff --git a/src/backend/src/filesystem/core/FSEntry.ts b/src/backend/src/filesystem/core/FSEntry.ts deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/src/backend/src/filesystem/core/Selector.ts b/src/backend/src/filesystem/core/Selector.ts deleted file mode 100644 index 82546c2ff6..0000000000 --- a/src/backend/src/filesystem/core/Selector.ts +++ /dev/null @@ -1,65 +0,0 @@ -import * as _path from 'path'; -import * as _util from 'util'; - -type TemporeryNodeType = any; - -export interface ISelector { - describe (showDebug?: boolean): string; - setPropertiesKnownBySelector (node: object): void; -} - -export class NodePathSelector { - public value: string; - - constructor (path: string) { - this.value = path; - } - - public describe (showDebug?: boolean): string { - return this.value; - } - - public setPropertiesKnownBySelector (node: TemporeryNodeType): void { - node.path = this.value; - node.name = _path.basename(this.value); - } -} - -export class NodeInternalUIDSelector { - public value: string; - - constructor (uid: string) { - this.value = uid; - } - - public describe (showDebug?: boolean): string { - return `[uid:${this.value}]`; - } - - public setPropertiesKnownBySelector (node: TemporeryNodeType): void { - node.uid = this.value; - } -} - -export class NodeInternalIDSelector { - constructor ( - public service: string, - public id: number, - public debugInfo: any - ) { } - - public describe (showDebug?: boolean): string { - if ( showDebug ) { - return `[db:${this.id}] (${ - _util.inspect(this.debugInfo) - })`; - } - return `[db:${this.id}]`; - } - - public setPropertiesKnownBySelector (node: TemporeryNodeType): void { - if ( this.service === 'mysql' ) { - node.id = this.id; - } - } -} diff --git a/src/backend/src/filesystem/hl_operations/hl_copy.js b/src/backend/src/filesystem/hl_operations/hl_copy.js index 6c48550cd2..d882e62fcb 100644 --- a/src/backend/src/filesystem/hl_operations/hl_copy.js +++ b/src/backend/src/filesystem/hl_operations/hl_copy.js @@ -141,8 +141,8 @@ class HLCopy extends HLFilesystemOperation { const sizeService = svc.get('sizeService'); let deset_usage = await sizeService.get_usage(dest_user.id); - const size = await source.fetchSize(values.user); - let capacity = config.is_storage_limited ? (dest_user.free_storage === undefined || dest_user.free_storage === null) ? config.storage_capacity : dest_user.free_storage : config.available_device_storage + const size = await source.fetchSize(); + const capacity = await sizeService.get_storage_capacity(dest_user.id); if(capacity - deset_usage - size < 0){ throw APIError.create('storage_limit_reached'); } diff --git a/src/backend/src/filesystem/hl_operations/hl_data_read.js b/src/backend/src/filesystem/hl_operations/hl_data_read.js index f7ec619c05..ff3d98ff10 100644 --- a/src/backend/src/filesystem/hl_operations/hl_data_read.js +++ b/src/backend/src/filesystem/hl_operations/hl_data_read.js @@ -75,14 +75,11 @@ class HLDataRead extends HLFilesystemOperation { const output_stream = new PassThrough(); - new Promise((resolve, reject) => { - rl.on('line', (line) => { - output_stream.write(line); - }); - rl.on('close', () => { - output_stream.end(); - resolve(); - }); + rl.on('line', (line) => { + output_stream.write(line); + }); + rl.on('close', () => { + output_stream.end(); }); return output_stream; diff --git a/src/backend/src/filesystem/hl_operations/hl_mkdir.js b/src/backend/src/filesystem/hl_operations/hl_mkdir.js index 648c783bef..7f3e4877dc 100644 --- a/src/backend/src/filesystem/hl_operations/hl_mkdir.js +++ b/src/backend/src/filesystem/hl_operations/hl_mkdir.js @@ -73,8 +73,7 @@ class MkTree extends HLFilesystemOperation { } async create_branch_ ({ parent_node, tree, parent_exists }) { - const { context, values } = this; - const { _path } = this.modules; + const { context } = this; const fs = context.get('services').get('filesystem'); const actor = context.get('actor'); @@ -82,7 +81,6 @@ class MkTree extends HLFilesystemOperation { const branches = tree.slice(1); let current = parent_node.selector; - let lastCreatedSelector = parent_node.selector; // trunk = a/b/c @@ -242,7 +240,6 @@ class HLMkdir extends HLFilesystemOperation { static MODULES = { _path: require('path'), - socketio: require('../../socketio.js'), } static PROPERTIES = { @@ -258,7 +255,7 @@ class HLMkdir extends HLFilesystemOperation { async _run () { const { context, values } = this; - const { _path, socketio } = this.modules; + const { _path } = this.modules; const fs = context.get('services').get('filesystem'); if ( ! is_valid_path(values.path, { @@ -385,12 +382,8 @@ class HLMkdir extends HLFilesystemOperation { } async _create_parents ({ parent_node }) { - const { context, values } = this; + const { values } = this; const { _path } = this.modules; - const fs = context.get('services').get('filesystem'); - - let current = parent_node.selector; - let lastCreatedSelector = null; const tree_op = new MkTree(); await tree_op.run({ diff --git a/src/backend/src/filesystem/hl_operations/hl_mklink.js b/src/backend/src/filesystem/hl_operations/hl_mklink.js index a24b20f012..f2fbe6446b 100644 --- a/src/backend/src/filesystem/hl_operations/hl_mklink.js +++ b/src/backend/src/filesystem/hl_operations/hl_mklink.js @@ -36,11 +36,10 @@ class HLMkLink extends HLFilesystemOperation { async _run () { const { context, values } = this; - const { _path } = this.modules; const fs = context.get('services').get('filesystem'); const { target, parent, user } = values; - let { name, dedupe_name } = values; + let { name } = values; if ( ! name ) { throw APIError.create('field_empty', null, { key: 'name' }); diff --git a/src/backend/src/filesystem/hl_operations/hl_mkshortcut.js b/src/backend/src/filesystem/hl_operations/hl_mkshortcut.js index 21a53b09a0..f917b10e0f 100644 --- a/src/backend/src/filesystem/hl_operations/hl_mkshortcut.js +++ b/src/backend/src/filesystem/hl_operations/hl_mkshortcut.js @@ -40,7 +40,6 @@ class HLMkShortcut extends HLFilesystemOperation { async _run () { console.log('HLMKSHORTCUT IS HAPPENING') const { context, values } = this; - const { _path, socketio } = this.modules; const fs = context.get('services').get('filesystem'); const { target, parent, user } = values; diff --git a/src/backend/src/filesystem/hl_operations/hl_move.js b/src/backend/src/filesystem/hl_operations/hl_move.js index 1fd4b801ca..dd66b2fe64 100644 --- a/src/backend/src/filesystem/hl_operations/hl_move.js +++ b/src/backend/src/filesystem/hl_operations/hl_move.js @@ -106,7 +106,8 @@ class HLMove extends HLFilesystemOperation { dest_user = source_user; await source.fetchSize(); const item_size = source.entry.size; - let capacity = config.is_storage_limited ? (dest_user.free_storage === undefined || dest_user.free_storage === null) ? config.storage_capacity : dest_user.free_storage : config.available_device_storage; + const sizeService = svc.get('sizeService'); + const capacity = await sizeService.get_storage_capacity(user.id); if(capacity - await df(dest_user.id) - item_size < 0){ throw APIError.create('storage_limit_reached'); } diff --git a/src/backend/src/filesystem/hl_operations/hl_read.js b/src/backend/src/filesystem/hl_operations/hl_read.js index 56de932a87..497efccf10 100644 --- a/src/backend/src/filesystem/hl_operations/hl_read.js +++ b/src/backend/src/filesystem/hl_operations/hl_read.js @@ -28,7 +28,6 @@ class HLRead extends HLFilesystemOperation { } async _run () { - const { context } = this; const { fsNode, actor, line_count, byte_count, diff --git a/src/backend/src/filesystem/hl_operations/hl_readdir.js b/src/backend/src/filesystem/hl_operations/hl_readdir.js index 32061f0284..ebf98e087f 100644 --- a/src/backend/src/filesystem/hl_operations/hl_readdir.js +++ b/src/backend/src/filesystem/hl_operations/hl_readdir.js @@ -81,13 +81,6 @@ class HLReadDir extends HLFilesystemOperation { await child.fetchSuggestedApps(user); await child.fetchSubdomains(user); } - const fs = require('fs'); - // fs.appendFileSync('/tmp/children.log', - // JSON.stringify({ - // no_thumbs, - // no_assocs, - // entry: child.entry, - // }) + '\n'); return await child.getSafeEntry({ thumbnail: ! no_thumbs }); })); } diff --git a/src/backend/src/filesystem/hl_operations/hl_write.js b/src/backend/src/filesystem/hl_operations/hl_write.js index 70b5b92987..2801580cea 100644 --- a/src/backend/src/filesystem/hl_operations/hl_write.js +++ b/src/backend/src/filesystem/hl_operations/hl_write.js @@ -23,7 +23,7 @@ const StringParam = require("../../api/filesystem/StringParam"); const UserParam = require("../../api/filesystem/UserParam"); const config = require("../../config"); const { chkperm, validate_fsentry_name } = require("../../helpers"); -const { TeePromise } = require("../../util/promise"); +const { TeePromise } = require("@heyputer/putility").libs.promise; const { pausing_tee, logging_stream, offset_write_stream, stream_to_the_void } = require("../../util/streamutil"); const { TYPE_DIRECTORY } = require("../FSNodeContext"); const { LLRead } = require("../ll_operations/ll_read"); @@ -65,8 +65,7 @@ class WriteCommonFeature { if ( ! user ) user = this.values.actor.type.user; const usage = await sizeService.get_usage(user.id); - let capacity = config.is_storage_limited ? user.free_storage == undefined - ? config.storage_capacity : user.free_storage : config.available_device_storage; + const capacity = await sizeService.get_storage_capacity(user.id); if( capacity - usage - file.size < 0 ) { throw APIError.create('storage_limit_reached'); } @@ -116,7 +115,6 @@ class HLWrite extends HLFilesystemOperation { static MODULES = { _path: require('path'), - socketio: require('../../socketio.js'), mime: require('mime-types'), } @@ -303,7 +301,7 @@ class HLWrite extends HLFilesystemOperation { this.checkpoint('before thumbnail'); let thumbnail_promise = new TeePromise(); - if ( await destination.isAppDataDirectory() ) { + if ( await destination.isAppDataDirectory() || values.no_thumbnail ) { thumbnail_promise.resolve(undefined); } else (async () => { const reason = await (async () => { diff --git a/src/backend/src/filesystem/ll_operations/ll_copy.js b/src/backend/src/filesystem/ll_operations/ll_copy.js index 406e90d5c2..47592d6789 100644 --- a/src/backend/src/filesystem/ll_operations/ll_copy.js +++ b/src/backend/src/filesystem/ll_operations/ll_copy.js @@ -21,7 +21,7 @@ const { Context } = require('../../util/context'); const { ParallelTasks } = require('../../util/otelutil'); const FSNodeContext = require('../FSNodeContext'); const { NodeUIDSelector } = require('../node/selectors'); -const { RESOURCE_STATUS_PENDING_CREATE } = require('../storage/ResourceService'); +const { RESOURCE_STATUS_PENDING_CREATE } = require('../../modules/puterfs/ResourceService'); const { UploadProgressTracker } = require('../storage/UploadProgressTracker'); const { LLFilesystemOperation } = require('./definitions'); @@ -157,15 +157,15 @@ class LLCopy extends LLFilesystemOperation { status: RESOURCE_STATUS_PENDING_CREATE, }); - const svc_fsentry = svc.get('systemFSEntryService'); + const svc_fsEntry = svc.get('fsEntryService'); this.log.info(`inserting entry: ` + uuid); - const entryOp = await svc_fsentry.insert(raw_fsentry); + const entryOp = await svc_fsEntry.insert(raw_fsentry); let node; this.checkpoint('before parallel tasks'); const tasks = new ParallelTasks({ tracer, max: 4 }); - await tracer.startActiveSpan(`fs:cp:parallel-portion`, async span => { + await Context.arun(`fs:cp:parallel-portion`, async () => { this.checkpoint('starting parallel tasks'); // Add child copy tasks if this is a directory if ( source.entry.is_dir ) { @@ -181,7 +181,6 @@ class LLCopy extends LLFilesystemOperation { const child_name = await child_node.get('name'); // TODO: this should be LLCopy instead const ll_copy = new LLCopy(); - console.log('LL Copy Start'); await ll_copy.run({ source: await fs.node( new NodeUIDSelector(child_uuid) @@ -192,30 +191,6 @@ class LLCopy extends LLFilesystemOperation { user, target_name: child_name, }); - console.log('LL Copy End'); - // const hl_copy = new HLCopy(); - // await hl_copy.run({ - // destination_or_parent: await fs.node( - // new NodeUIDSelector(uuid) - // ), - // source: await fs.node( - // new NodeUIDSelector(child_uuid) - // ), - // user - // }); - // await fs.cp(fs, { - // source: await fs.node( - // new NodeUIDSelector(child_uuid) - // ), - // // TODO: don't do this when cp supports uuids - // destinationOrParent: await fs.node( - // new NodeUIDSelector(uuid) - // ), - // user, - // overwrite: false, - // create_missing_parents: false, - // ancestor_check_not_needed: true, - // }); }); } } @@ -241,8 +216,6 @@ class LLCopy extends LLFilesystemOperation { this.checkpoint('waiting for parallel tasks'); await tasks.awaitAll(); this.checkpoint('finishing up'); - - span.end(); }); node = node || await fs.node(new NodeUIDSelector(uuid)); diff --git a/src/backend/src/filesystem/ll_operations/ll_mkdir.js b/src/backend/src/filesystem/ll_operations/ll_mkdir.js index 005df32432..1ba37f99f7 100644 --- a/src/backend/src/filesystem/ll_operations/ll_mkdir.js +++ b/src/backend/src/filesystem/ll_operations/ll_mkdir.js @@ -21,7 +21,7 @@ const { MODE_WRITE } = require("../../services/fs/FSLockService"); const { Context } = require("../../util/context"); const { TYPE_DIRECTORY } = require("../FSNodeContext"); const { NodeUIDSelector, NodeChildSelector } = require("../node/selectors"); -const { RESOURCE_STATUS_PENDING_CREATE } = require("../storage/ResourceService"); +const { RESOURCE_STATUS_PENDING_CREATE } = require("../../modules/puterfs/ResourceService"); const { LLFilesystemOperation } = require("./definitions"); class LLMkdir extends LLFilesystemOperation { @@ -60,7 +60,7 @@ class LLMkdir extends LLFilesystemOperation { const ts = Math.round(Date.now() / 1000); const uid = uuidv4(); const resourceService = context.get('services').get('resourceService'); - const systemFSEntryService = context.get('services').get('systemFSEntryService'); + const svc_fsEntry = context.get('services').get('fsEntryService'); const svc_event = context.get('services').get('event'); const fs = context.get('services').get('filesystem'); @@ -109,7 +109,7 @@ class LLMkdir extends LLFilesystemOperation { this.log.debug('creating fsentry', { fsentry: raw_fsentry }) this.checkpoint('about to enqueue insert'); - const entryOp = await systemFSEntryService.insert(raw_fsentry); + const entryOp = await svc_fsEntry.insert(raw_fsentry); this.field('fsentry-created', false); diff --git a/src/backend/src/filesystem/ll_operations/ll_read.js b/src/backend/src/filesystem/ll_operations/ll_read.js index b903a9dd1d..1785062f01 100644 --- a/src/backend/src/filesystem/ll_operations/ll_read.js +++ b/src/backend/src/filesystem/ll_operations/ll_read.js @@ -17,7 +17,6 @@ * along with this program. If not, see . */ const APIError = require("../../api/APIError"); -const { CodeModel } = require("../../codex/CodeModel"); const { Sequence } = require("../../codex/Sequence"); const { DB_WRITE } = require("../../services/database/consts"); diff --git a/src/backend/src/filesystem/ll_operations/ll_readdir.js b/src/backend/src/filesystem/ll_operations/ll_readdir.js index 5a07dcbb13..83defc4836 100644 --- a/src/backend/src/filesystem/ll_operations/ll_readdir.js +++ b/src/backend/src/filesystem/ll_operations/ll_readdir.js @@ -25,7 +25,7 @@ const { LLFilesystemOperation } = require("./definitions"); class LLReadDir extends LLFilesystemOperation { async _run () { const { context } = this; - const { subject: subject_let, user, actor, no_acl } = this.values; + const { subject: subject_let, actor, no_acl } = this.values; let subject = subject_let; if ( ! await subject.exists() ) { diff --git a/src/backend/src/filesystem/ll_operations/ll_readshares.js b/src/backend/src/filesystem/ll_operations/ll_readshares.js index c797d65f0a..1dd8892712 100644 --- a/src/backend/src/filesystem/ll_operations/ll_readshares.js +++ b/src/backend/src/filesystem/ll_operations/ll_readshares.js @@ -36,7 +36,7 @@ class LLReadShares extends LLFilesystemOperation { `; async _run () { - const { subject, user, actor, depth = 0 } = this.values; + const { subject, user, actor } = this.values; const svc = this.context.get('services'); diff --git a/src/backend/src/filesystem/ll_operations/ll_write.js b/src/backend/src/filesystem/ll_operations/ll_write.js index 9b16fa20fc..4e1eb4d4b6 100644 --- a/src/backend/src/filesystem/ll_operations/ll_write.js +++ b/src/backend/src/filesystem/ll_operations/ll_write.js @@ -18,7 +18,7 @@ */ const { Context } = require("../../util/context"); const { LLFilesystemOperation } = require("./definitions"); -const { RESOURCE_STATUS_PENDING_CREATE } = require("../storage/ResourceService"); +const { RESOURCE_STATUS_PENDING_CREATE } = require("../../modules/puterfs/ResourceService.js"); const { NodeUIDSelector } = require("../node/selectors"); const { UploadProgressTracker } = require("../storage/UploadProgressTracker"); const FSNodeContext = require("../FSNodeContext"); @@ -136,7 +136,7 @@ class LLOWrite extends LLWriteBase { const svc = Context.get('services'); const sizeService = svc.get('sizeService'); const resourceService = svc.get('resourceService'); - const systemFSEntryService = svc.get('systemFSEntryService'); + const svc_fsEntry = svc.get('fsEntryService'); const svc_event = svc.get('event'); // TODO: fs:decouple-versions @@ -188,7 +188,7 @@ class LLOWrite extends LLWriteBase { const filesize = file.size; sizeService.change_usage(actor.type.user.id, filesize); - const entryOp = await systemFSEntryService.update(uid, raw_fsentry_delta); + const entryOp = await svc_fsEntry.update(uid, raw_fsentry_delta); // depends on fsentry, does not depend on S3 (async () => { @@ -235,7 +235,7 @@ class LLCWrite extends LLWriteBase { const svc = Context.get('services'); const sizeService = svc.get('sizeService'); const resourceService = svc.get('resourceService'); - const systemFSEntryService = svc.get('systemFSEntryService'); + const svc_fsEntry = svc.get('fsEntryService'); const svc_event = svc.get('event'); const fs = svc.get('filesystem'); @@ -317,7 +317,7 @@ class LLCWrite extends LLWriteBase { this.checkpoint('after change_usage'); - const entryOp = await systemFSEntryService.insert(raw_fsentry); + const entryOp = await svc_fsEntry.insert(raw_fsentry); this.checkpoint('after fsentry insert enqueue'); diff --git a/src/backend/src/filesystem/storage/SystemFSEntryService.js b/src/backend/src/filesystem/storage/SystemFSEntryService.js deleted file mode 100644 index a226602535..0000000000 --- a/src/backend/src/filesystem/storage/SystemFSEntryService.js +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright (C) 2024 Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ -const { PuterPath } = require("../lib/PuterPath"); -const _path = require('path'); - -// Redis keys: -// ::::: -// -// note: is added by redisService automatically. -// -// If `` is `multi`, then the format differs slightly: -// :::multi:::: -// where `` specifies the property being used for the id - -class SystemFSEntryService { - constructor ({ services }) { - this.redis = { enabled: false }; - this.DatabaseFSEntryService = services.get('fsEntryService'); - - this.log = services.get('log-service').create('system-fsentry-service'); - - // Register information providers - const info = services.get('information'); - this.info = info; - - if ( ! this.redis.enabled ) return; - - // path -> uuid via redis - info.given('fs.fsentry:path').provide('fs.fsentry:uuid') - .addStrategy('redis', async path => { - return await this.get_uuid_from_path(path); - }); - // uuid -> path via redis - info.given('fs.fsentry:uuid').provide('fs.fsentry:path') - .addStrategy('redis', async uuid => { - this.log.debug('getting path for: ' + uuid); - if ( uuid === PuterPath.NULL_UUID ) return '/'; - const res = ( await this.redis.get(`fs:fsentry:path:path:${uuid}`) ) ?? undefined; - this.log.debug('got path: ' + res); - return res; - }); - // uuid -> parent_uuid via redis - info.given('fs.fsentry:uuid').provide('fs.fsentry:children(fs.fsentry:uuid)') - .addStrategy('redis', async uuid => { - return await this.get_child_uuids(uuid); - }); - } - - async insert (entry) { - if ( this.redis.enabled ) { - await this._link(entry.uuid, entry.parent_uid, entry.name); - } - return await this.DatabaseFSEntryService.insert(entry); - } - - async update (uuid, entry) { - // If parent_uid is set during an update, we assume that it - // has been changed. If it hasn't, no problem: just an extra - // cache invalidation; but the code that set it should know - // better because it probably has the fsentry data already. - if ( entry.hasOwnProperty('parent_uid') ) { - await this._relocate(uuid, entry.parent_uid) - } - return await this.DatabaseFSEntryService.update(uuid, entry); - } - - async delete (uuid) { - // - } - - async get_child_uuids (uuid) { - let members; - members = await this.redis.smembers(`fs:fsentry:set:childs:${uuid}`); - if ( members ) return members; - members = await this.DatabaseFSEntryService.get_descendants(uuid); - return members ?? []; - } - - async get_uuid_from_path (path) { - path = PuterPath.adapt(path); - - let current = path.reference; - let pathOfReference = path.reference === PuterPath.NULL_UUID - ? '/' : this.get_path_from_uuid(path.reference); - - const fullPath = _path.join(pathOfReference, path.relativePortion); - let uuid = await this.redis.get(`fs:fsentry:multi:uuid:uuid:path:${fullPath}`); - return uuid; - } - - // Cache related functions - async _link (subject_uuid, parent_uuid, subject_name) { - this.log.info(`linking ${subject_uuid} to ${parent_uuid}`); - // We need the parent's path to update everything - - let pathOfParent = await this.info.with('fs.fsentry:uuid') - .obtain('fs.fsentry:path').exec(parent_uuid); - - this.log.debug(`path of parent: ${pathOfParent}`); - - if ( ! subject_name ) { - subject_name = await this.redis.get(`fs:fsentry:str:name:${subject_uuid}`); - } - - // Register properties - await this.redis.set(`fs:fsentry:uuid:parent:${subject_uuid}`, parent_uuid); - await this.redis.set(`fs:fsentry:str:name:${subject_uuid}`, subject_name); - - // Add as child of parent - await this.redis.sadd(`fs:fsentry:set:childs:${parent_uuid}`, subject_uuid); - - // Register path - const subject_path = `${pathOfParent}/${subject_name}`; - this.log.debug(`registering path: ${subject_path} for ${subject_uuid}`); - await this.redis.set(`fs:fsentry:path:path:${subject_uuid}`, subject_path); - await this.redis.set(`fs:fsentry:multi:uuid:uuid:path:${subject_path}`, subject_uuid); - } - - async _unlink (subject_uuid) { - let parent_uuid = await this.redis.get(`fs:fsentry:uuid:parent:${subject_uuid}`); - // TODO: try getting from database - - // Remove from parent - await this.redis.srem(`fs:fsentry:set:childs:${parent_uuid}`, subject_uuid); - } - - async _purge (subject_uuid) { - await this._unlink(subject_uuid); - - // Remove properties - await this.redis.del(`fs:fsentry:uuid:parent:${subject_uuid}`); - await this.redis.del(`fs:fsentry:str:name:${subject_uuid}`); - - // Remove path - const subject_path = - await this.redis.get(`fs:fsentry:path:path:${subject_uuid}`); - await this.redis.del(`fs:fsentry:path:path:${subject_uuid}`); - if ( subject_path ) { - await this.redis.del(`fs:fsentry:multi:uuid:path:${subject_path}`); - } - } - - async _relocate (subject_uuid, new_parent_uuid) { - await this._unlink(subject_uuid); - await this._link(subject_uuid, new_parent_uuid); - } -} - -module.exports = SystemFSEntryService; diff --git a/src/backend/src/fun/dev-console-ui-utils.js b/src/backend/src/fun/dev-console-ui-utils.js index 102370a1ab..27c5392b70 100644 --- a/src/backend/src/fun/dev-console-ui-utils.js +++ b/src/backend/src/fun/dev-console-ui-utils.js @@ -17,7 +17,7 @@ * along with this program. If not, see . */ const config = require('../config'); -const { TeePromise } = require('../util/promise'); +const { TeePromise } = require('@heyputer/putility').libs.promise; const es_import_promise = new TeePromise(); let stringLength; diff --git a/src/backend/src/helpers.js b/src/backend/src/helpers.js index c9de4a8db3..51cc0da9bf 100644 --- a/src/backend/src/helpers.js +++ b/src/backend/src/helpers.js @@ -22,7 +22,6 @@ const micromatch = require('micromatch'); const config = require('./config') const mime = require('mime-types'); const PerformanceMonitor = require('./monitor/PerformanceMonitor.js'); -const { generate_identifier } = require('./util/identifier.js'); const { ManagedError } = require('./util/errorutil.js'); const { spanify } = require('./util/otelutil.js'); const APIError = require('./api/APIError.js'); @@ -1012,7 +1011,6 @@ async function gen_public_token(file_uuid, ttl = 24 * 60 * 60){ } const uid = fsentry.uuid; - const expires = Math.ceil(Date.now() / 1000) + ttl; const token = uuidv4(); const contentType = mime.contentType(fsentry.name); @@ -1117,6 +1115,7 @@ async function jwt_auth(req){ } return { + actor, user: actor.type.user, token: token, }; @@ -1155,201 +1154,6 @@ async function jwt_auth(req){ return ancestors; } -// THIS LEGACY FUNCTION IS STILL IN USE -// by: generate_system_fsentries -// TODO: migrate generate_system_fsentries to use QuickMkdir -async function mkdir(options){ - const fs = systemfs; - - debugger; - - const resolved_path = PathBuilder.resolve(options.path, { puterfs: true }); - - const dirpath = _path.dirname(resolved_path); - let target_name = _path.basename(resolved_path); - const overwrite = options.overwrite ?? false; - const dedupe_name = options.dedupe_name ?? false; - const immutable = options.immutable ?? false; - const return_id = options.return_id ?? false; - const no_perm_check = options.no_perm_check ?? false; - - // make parent directories as needed - const create_missing_parents = options.create_missing_parents ?? false; - - // hold a list of all parent directories created in the process - let parent_dirs_created = []; - let overwritten_uid; - - // target_name validation - try{ - validate_fsentry_name(target_name) - }catch(e){ - throw e.message; - } - - // resolve dirpath to its fsentry - let parent = await convert_path_to_fsentry(dirpath); - - // dirpath not found - if(parent === false && !create_missing_parents) - throw new Error("Target path not found"); - // create missing parent directories - else if(parent === false && create_missing_parents){ - const dirs = _path.resolve('/', dirpath).split('/'); - let cur_path = ''; - for(let j=0; j < dirs.length; j++){ - if(dirs[j] === '') - continue; - - cur_path += '/'+dirs[j]; - // skip creating '/[username]' - if(j === 1) - continue; - try{ - let d = await mkdir(fs, {path: cur_path, user: options.user}); - d.path = cur_path; - parent_dirs_created.push(d); - }catch(e){ - console.log(`Skipped mkdir ${cur_path}`); - } - } - // try setting parent again - parent = await convert_path_to_fsentry(dirpath); - if(parent === false) - throw new Error("Target path not found"); - } - - // check permission - if(!no_perm_check && !await chkperm(parent, options.user.id, 'write')) - throw { code:`forbidden`, message: `permission denied.`}; - - // check if a fsentry with the same name exists under this path - const existing_fsentry = await convert_path_to_fsentry(_path.resolve('/', dirpath + '/' + target_name )); - - /** @type BaseDatabaseAccessService */ - const db = services.get('database').get(DB_WRITE, 'filesystem'); - - // if trying to create a directory with an existing path and overwrite==false, throw an error - if(!overwrite && !dedupe_name && existing_fsentry !== false){ - throw { - code: 'path_exists', - message:"A file/directory with the same path already exists.", - entry_name: existing_fsentry.name, - existing_fsentry: { - name: existing_fsentry.name, - uid: existing_fsentry.uuid, - } - }; - } - else if(overwrite && existing_fsentry){ - overwritten_uid = existing_fsentry.uuid; - // check permission - if(!await chkperm(existing_fsentry, options.user.id, 'write')) - throw {code:`forbidden`, message: `permission denied.`}; - // delete existing dir - await db.write( - `DELETE FROM fsentries WHERE id = ? AND user_id = ?`, - [ - //parent_uid - existing_fsentry.uuid, - //user_id - options.user.id, - ]); - } - // dedupe name, generate a new name until its unique - else if(dedupe_name && existing_fsentry !== false){ - for( let i = 1; ; i++){ - let try_new_name = existing_fsentry.name + ' (' + i + ')'; - let check_dupe = await db.read( - "SELECT `id` FROM `fsentries` WHERE `parent_uid` = ? AND name = ? LIMIT 1", - [existing_fsentry.parent_uid, try_new_name] - ); - if(check_dupe[0] === undefined){ - target_name = try_new_name; - break; - } - } - } - - // shrotcut? - let shortcut_fsentry; - if(options.shortcut_to){ - shortcut_fsentry = await uuid2fsentry(options.shortcut_to); - if(shortcut_fsentry === false){ - throw ({ code:`not_found`, message: `shortcut_to not found.`}) - }else if(!parent.is_dir){ - throw ({ code:`not_dir`, message: `parent of shortcut_to must be a directory`}) - }else if(!await chkperm(shortcut_fsentry, options.user.id, 'read')){ - throw ({ code:`forbidden`, message: `shortcut_to permission denied.`}) - } - } - - // current epoch - const ts = Math.round(Date.now() / 1000) - const uid = uuidv4(); - - // record in db - let user_id = (parent === null ? options.user.id : parent.user_id); - const { insertId: mkdir_db_id } = await db.write( - `INSERT INTO fsentries - (uuid, parent_uid, user_id, name, is_dir, created, modified, immutable, shortcut_to, is_shortcut) VALUES - ( ?, ?, ?, ?, true, ?, ?, ?, ?, ?)`, - [ - //uuid - uid, - //parent_uid - (parent === null) ? null : parent.uuid, - //user_id - user_id, - //name - target_name, - //created - ts, - //modified - ts, - //immutable - immutable, - //shortcut_to, - shortcut_fsentry ? shortcut_fsentry.id : null, - //is_shortcut, - shortcut_fsentry ? 1 : 0, - ] - ); - - const ret_obj = { - uid : uid, - name: target_name, - immutable: immutable, - is_dir: true, - path: options.path ?? false, - dirpath: dirpath, - is_shared: await is_shared_with_anyone(mkdir_db_id), - overwritten_uid: overwritten_uid, - shortcut_to: shortcut_fsentry ? shortcut_fsentry.uuid : null, - shortcut_to_path: shortcut_fsentry ? await id2path(shortcut_fsentry.id) : null, - parent_dirs_created: parent_dirs_created, - original_client_socket_id: options.original_client_socket_id, - }; - // add existing_fsentry if exists - if(existing_fsentry){ - ret_obj.existing_fsentry ={ - name: existing_fsentry.name, - uid: existing_fsentry.uuid, - } - } - - if(return_id) - ret_obj.id = mkdir_db_id; - - // send realtime success msg to client - let socketio = require('./socketio.js').getio(); - if(socketio){ - socketio.to(user_id).emit('item.added', ret_obj) - } - - return ret_obj; -} - function is_valid_uuid ( uuid ) { let s = "" + uuid; s = s.match(/^[0-9a-f]{8}-[0-9a-f]{4}-[0-5][0-9a-f]{3}-[089ab][0-9a-f]{3}-[0-9a-f]{12}$/i); @@ -1412,100 +1216,6 @@ async function app_name_exists(name){ return true; } - -// generates all the default files and directories a user needs, -// generally used for a brand new account -async function generate_system_fsentries(user){ - /** @type BaseDatabaseAccessService */ - const db = services.get('database').get(DB_WRITE, 'filesystem'); - - //------------------------------------------------------------- - // create root `/[username]/` - //------------------------------------------------------------- - const root_dir = await mkdir({ - path: '/' + user.username, - user: user, - immutable: true, - no_perm_check: true, - return_id: true, - }); - - // Normally, it is recommended to use mkdir() to create new folders, - // but during signup this could result in multiple queries to the DB server - // and for servers in remote regions such as Asia this could result in a - // very long time for /signup to finish, sometimes up to 30-40 seconds! - // by combining as many queries as we can into one and avoiding multiple back-and-forth - // with the DB server, we can speed this process up significantly. - const ts = Date.now()/1000; - - // Generate UUIDs for all the default folders and files - let trash_uuid = uuidv4(); - let appdata_uuid = uuidv4(); - let desktop_uuid = uuidv4(); - let documents_uuid = uuidv4(); - let pictures_uuid = uuidv4(); - let videos_uuid = uuidv4(); - let public_uuid = uuidv4(); - - const insert_res = await db.write( - `INSERT INTO fsentries - (uuid, parent_uid, user_id, name, path, is_dir, created, modified, immutable) VALUES - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true), - ( ?, ?, ?, ?, ?, true, ?, ?, true) - `, - [ - // Trash - trash_uuid, root_dir.uid, user.id, 'Trash', `/${user.username}/Trash`, ts, ts, - // AppData - appdata_uuid, root_dir.uid, user.id, 'AppData', `/${user.username}/AppData`, ts, ts, - // Desktop - desktop_uuid, root_dir.uid, user.id, 'Desktop', `/${user.username}/Desktop`, ts, ts, - // Documents - documents_uuid, root_dir.uid, user.id, 'Documents', `/${user.username}/Documents`, ts, ts, - // Pictures - pictures_uuid, root_dir.uid, user.id, 'Pictures', `/${user.username}/Pictures`, ts, ts, - // Videos - videos_uuid, root_dir.uid, user.id, 'Videos', `/${user.username}/Videos`, ts, ts, - // Public - public_uuid, root_dir.uid, user.id, 'Public', `/${user.username}/Public`, ts, ts, - ] - ); - - // https://stackoverflow.com/a/50103616 - let trash_id = insert_res.insertId; - let appdata_id = insert_res.insertId + 1; - let desktop_id = insert_res.insertId + 2; - let documents_id = insert_res.insertId + 3; - let pictures_id = insert_res.insertId + 4; - let videos_id = insert_res.insertId + 5; - let public_id = insert_res.insertId + 6; - - // Asynchronously set the user's system folders uuids in database - // This is for caching purposes, so we don't have to query the DB every time we need to access these folders - // This is also possible because we know the user's system folders uuids will never change - - // TODO: pass to IIAFE manager to avoid unhandled promise rejection - // (IIAFE manager doesn't exist yet, hence this is a TODO) - db.write( - `UPDATE user SET - trash_uuid=?, appdata_uuid=?, desktop_uuid=?, documents_uuid=?, pictures_uuid=?, videos_uuid=?, public_uuid=?, - trash_id=?, appdata_id=?, desktop_id=?, documents_id=?, pictures_id=?, videos_id=?, public_id=? - - WHERE id=?`, - [ - trash_uuid, appdata_uuid, desktop_uuid, documents_uuid, pictures_uuid, videos_uuid, public_uuid, - trash_id, appdata_id, desktop_id, documents_id, pictures_id, videos_id, public_id, - user.id - ] - ); - invalidate_cached_user(user); -} - function send_email_verification_code(email_confirm_code, email){ const svc_email = Context.get('services').get('email'); svc_email.send_email({ email }, 'email_verification_code', { @@ -1519,19 +1229,11 @@ function send_email_verification_token(email_confirm_token, email, user_uuid){ svc_email.send_email({ email }, 'email_verification_link', { link }); } -async function generate_random_username(){ - let username; - do { - username = generate_identifier(); - } while (await username_exists(username)); - return username; -} - function generate_random_str(length) { - var result = ''; - var characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; - var charactersLength = characters.length; - for ( var i = 0; i < length; i++ ) { + let result = ''; + const characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; + const charactersLength = characters.length; + for ( let i = 0; i < length; i++ ) { result += characters.charAt(Math.floor(Math.random() * charactersLength)); } @@ -1546,11 +1248,11 @@ function generate_random_str(length) { * @throws {TypeError} If the `seconds` parameter is not a number. */ function seconds_to_string(seconds) { - var numyears = Math.floor(seconds / 31536000); - var numdays = Math.floor((seconds % 31536000) / 86400); - var numhours = Math.floor(((seconds % 31536000) % 86400) / 3600); - var numminutes = Math.floor((((seconds % 31536000) % 86400) % 3600) / 60); - var numseconds = (((seconds % 31536000) % 86400) % 3600) % 60; + const numyears = Math.floor(seconds / 31536000); + const numdays = Math.floor((seconds % 31536000) / 86400); + const numhours = Math.floor(((seconds % 31536000) % 86400) / 3600); + const numminutes = Math.floor((((seconds % 31536000) % 86400) % 3600) / 60); + const numseconds = (((seconds % 31536000) % 86400) % 3600) % 60; return numyears + " years " + numdays + " days " + numhours + " hours " + numminutes + " minutes " + numseconds + " seconds"; } @@ -1565,14 +1267,11 @@ async function suggest_app_for_fsentry(fsentry, options){ const suggested_apps = []; let content_type = mime.contentType(fsentry.name); - if(content_type === null || content_type === undefined || content_type === false) - content_type = ''; + if( ! content_type ) content_type = ''; // IIFE just so fsname can stay `const` const fsname = (() => { if ( ! fsentry.name ) { - const fs = require('fs'); - fs.writeFileSync('/tmp/missing-fsentry-name.txt', JSON.stringify(fsentry, null, 2)); return 'missing-fsentry-name'; } let fsname = fsentry.name.toLowerCase(); @@ -1581,74 +1280,79 @@ async function suggest_app_for_fsentry(fsentry, options){ return fsname; })(); const file_extension = _path.extname(fsname).toLowerCase(); + + const any_of = (list, name) => { + return list.some(v => name.endsWith(v)); + } //--------------------------------------------- // Code //--------------------------------------------- - if( - fsname.endsWith('.asm') || - fsname.endsWith('.asp') || - fsname.endsWith('.aspx') || - fsname.endsWith('.bash') || - fsname.endsWith('.c') || - fsname.endsWith('.cpp') || - fsname.endsWith('.css') || - fsname.endsWith('.csv') || - fsname.endsWith('.dhtml') || - fsname.endsWith('.f') || - fsname.endsWith('.go') || - fsname.endsWith('.h') || - fsname.endsWith('.htm') || - fsname.endsWith('.html') || - fsname.endsWith('.html5') || - fsname.endsWith('.java') || - fsname.endsWith('.jl') || - fsname.endsWith('.js') || - fsname.endsWith('.jsa') || - fsname.endsWith('.json') || - fsname.endsWith('.jsonld') || - fsname.endsWith('.jsf') || - fsname.endsWith('.jsp') || - fsname.endsWith('.kt') || - fsname.endsWith('.log') || - fsname.endsWith('.lock') || - fsname.endsWith('.lua') || - fsname.endsWith('.md') || - fsname.endsWith('.perl') || - fsname.endsWith('.phar') || - fsname.endsWith('.php') || - fsname.endsWith('.pl') || - fsname.endsWith('.py') || - fsname.endsWith('.r') || - fsname.endsWith('.rb') || - fsname.endsWith('.rdata') || - fsname.endsWith('.rda') || - fsname.endsWith('.rdf') || - fsname.endsWith('.rds') || - fsname.endsWith('.rs') || - fsname.endsWith('.rlib') || - fsname.endsWith('.rpy') || - fsname.endsWith('.scala') || - fsname.endsWith('.sc') || - fsname.endsWith('.scm') || - fsname.endsWith('.sh') || - fsname.endsWith('.sol') || - fsname.endsWith('.sql') || - fsname.endsWith('.ss') || - fsname.endsWith('.svg') || - fsname.endsWith('.swift') || - fsname.endsWith('.toml') || - fsname.endsWith('.ts') || - fsname.endsWith('.wasm') || - fsname.endsWith('.xhtml') || - fsname.endsWith('.xml') || - fsname.endsWith('.yaml') || - // files with no extension - !fsname.includes('.') - ){ + const exts_code = [ + '.asm', + '.asp', + '.aspx', + '.bash', + '.c', + '.cpp', + '.css', + '.csv', + '.dhtml', + '.f', + '.go', + '.h', + '.htm', + '.html', + '.html5', + '.java', + '.jl', + '.js', + '.jsa', + '.json', + '.jsonld', + '.jsf', + '.jsp', + '.kt', + '.log', + '.lock', + '.lua', + '.md', + '.perl', + '.phar', + '.php', + '.pl', + '.py', + '.r', + '.rb', + '.rdata', + '.rda', + '.rdf', + '.rds', + '.rs', + '.rlib', + '.rpy', + '.scala', + '.sc', + '.scm', + '.sh', + '.sol', + '.sql', + '.ss', + '.svg', + '.swift', + '.toml', + '.ts', + '.wasm', + '.xhtml', + '.xml', + '.yaml', + ]; + + if ( any_of(exts_code, fsname) || !fsname.includes('.') ) { suggested_apps.push(await get_app({name: 'code'})) suggested_apps.push(await get_app({name: 'editor'})) } + //--------------------------------------------- // Editor //--------------------------------------------- @@ -1712,19 +1416,17 @@ async function suggest_app_for_fsentry(fsentry, options){ //--------------------------------------------- // 3rd-party apps //--------------------------------------------- - const apps = kv.get(`assocs:${file_extension.slice(1)}:apps`) + const apps = kv.get(`assocs:${file_extension.slice(1)}:apps`) ?? []; monitor.label("third party associations"); - if(apps && apps.length > 0){ - for (let index = 0; index < apps.length; index++) { - // retrieve app from DB - const third_party_app = await get_app({id: apps[index]}) - if ( ! third_party_app ) continue; - // only add if the app is approved for opening items or the app is owned by this user - if( third_party_app.approved_for_opening_items || - (options !== undefined && options.user !== undefined && options.user.id === third_party_app.owner_user_id)) - suggested_apps.push(third_party_app) - } + for ( const app_id of apps ) { + // retrieve app from DB + const third_party_app = await get_app({id: app_id}) + if ( ! third_party_app ) continue; + // only add if the app is approved for opening items or the app is owned by this user + if( third_party_app.approved_for_opening_items || + (options !== undefined && options.user !== undefined && options.user.id === third_party_app.owner_user_id)) + suggested_apps.push(third_party_app) } monitor.stamp(); monitor.end(); @@ -1741,10 +1443,6 @@ async function suggest_app_for_fsentry(fsentry, options){ }); } -function build_item_object(item){ - -} - async function get_taskbar_items(user) { /** @type BaseDatabaseAccessService */ const db = services.get('database').get(DB_WRITE, 'filesystem'); @@ -1813,7 +1511,7 @@ async function get_taskbar_items(user) { return taskbar_items; } -function validate_signature_auth(url, action) { +function validate_signature_auth(url, action, options = {}) { const query = new URL(url).searchParams; if(!query.get('uid')) @@ -1824,6 +1522,12 @@ function validate_signature_auth(url, action) { throw {message: '`expires` is required for signature-based authentication.'} else if(!query.get('signature')) throw {message: '`signature` is required for signature-based authentication.'} + + if ( options.uid ) { + if ( query.get('uid') !== options.uid ) { + throw {message: 'Authentication failed. `uid` does not match.'} + } + } const expired = query.get('expires') && (query.get('expires') < Date.now() / 1000); @@ -1867,13 +1571,13 @@ async function mv(options){ function number_format (number, decimals, dec_point, thousands_sep) { // Strip all characters but numerical ones. number = (number + '').replace(/[^0-9+\-Ee.]/g, ''); - var n = !isFinite(+number) ? 0 : +number, + let n = !isFinite(+number) ? 0 : +number, prec = !isFinite(+decimals) ? 0 : Math.abs(decimals), sep = (typeof thousands_sep === 'undefined') ? ',' : thousands_sep, dec = (typeof dec_point === 'undefined') ? '.' : dec_point, s = '', toFixedFix = function (n, prec) { - var k = Math.pow(10, prec); + const k = Math.pow(10, prec); return '' + Math.round(n * k) / k; }; // Fix for IE parseFloat(0.55).toFixed(0) = 0; @@ -1893,7 +1597,6 @@ module.exports = { app_name_exists, app_exists, body_parser_error_handler, - build_item_object, byte_format, change_username, chkperm, @@ -1905,9 +1608,7 @@ module.exports = { gen_public_token, get_taskbar_items, get_url_from_req, - generate_system_fsentries, generate_random_str, - generate_random_username, get_app, get_user, invalidate_cached_user, @@ -1926,7 +1627,6 @@ module.exports = { is_specifically_uuidv4, is_valid_url, jwt_auth, - mkdir, mv, number_format, refresh_apps_cache, diff --git a/src/backend/src/middleware/abuse.js b/src/backend/src/middleware/abuse.js index 10d4b7faee..59ee6ad928 100644 --- a/src/backend/src/middleware/abuse.js +++ b/src/backend/src/middleware/abuse.js @@ -20,7 +20,6 @@ const APIError = require("../api/APIError"); const { Context } = require("../util/context"); const abuse = options => (req, res, next) => { - // const svc_abuse = x.get('services').get('abuse-prevention'); const requester = Context.get('requester'); if ( options.no_bots ) { diff --git a/src/backend/src/services/runtime-analysis/AlarmService.js b/src/backend/src/modules/core/AlarmService.js similarity index 70% rename from src/backend/src/services/runtime-analysis/AlarmService.js rename to src/backend/src/modules/core/AlarmService.js index 718314b78f..a8cb2f9597 100644 --- a/src/backend/src/services/runtime-analysis/AlarmService.js +++ b/src/backend/src/modules/core/AlarmService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"openai-completion","model":"gpt-4o"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -23,55 +24,59 @@ const util = require('util'); const _path = require('path'); const fs = require('fs'); -const { fallbackRead } = require('../../util/files.js'); -const { generate_identifier } = require('../../util/identifier.js'); -const { stringify_log_entry } = require('./LogService.js'); -const BaseService = require('../BaseService.js'); -const { split_lines } = require('../../util/stdioutil.js'); -const { Context } = require('../../util/context.js'); +const BaseService = require('../../services/BaseService.js'); + +/** + * AlarmService class is responsible for managing alarms. + * It provides methods for creating, clearing, and handling alarms. + */ class AlarmService extends BaseService { + static USE = { + logutil: 'core.util.logutil', + identutil: 'core.util.identutil', + stdioutil: 'core.util.stdioutil', + Context: 'core.context', + } + /** + * This method initializes the AlarmService by setting up its internal data structures and initializing any required dependencies. + * + * It reads in the known errors from a JSON5 file and sets them as the known_errors property of the AlarmService instance. + * + * It also registers commands with the provided commands service. + */ async _construct () { this.alarms = {}; this.alarm_aliases = {}; this.known_errors = []; } + /** + * Method to initialize AlarmService. Sets the known errors and registers commands. + * @returns {Promise} + */ async _init () { const services = this.services; this.pager = services.get('pager'); // TODO:[self-hosted] fix this properly this.known_errors = []; - // (async () => { - // try { - // this.known_errors = JSON5.parse( - // await fallbackRead( - // 'data/known_errors.json5', - // '/var/puter/data/known_errors.json5', - // ), - // ); - // } catch (e) { - // this.create( - // 'missing-known-errors', - // e.message, - // ) - // } - // })(); - - this._register_commands(services.get('commands')); if ( this.global_config.env === 'dev' ) { + /** + * This method initializes the AlarmService instance by registering commands, setting up the pager, and initializing the known errors. + * It also sets up the widget to display alarms in the dev environment. + * + * @param {BaseService} services - The BaseService instance that provides access to other services. + * @returns {void} + */ this.alarm_widget = () => { - // return `\x1B[31;1m alarms (${ - // Object.keys(this.alarms) - // })\x1B[0m`; const lines = []; for ( const alarm of Object.values(this.alarms) ) { const line = `\x1B[31;1m [alarm]\x1B[0m ` + `${alarm.id_string}: ${alarm.message} (${alarm.count})`; - const line_lines = split_lines(line); + const line_lines = this.stdioutil.split_lines(line); lines.push(...line_lines); } @@ -79,27 +84,46 @@ class AlarmService extends BaseService { } } } + + /** + * AlarmService registers its commands at the consolidation phase because + * the '_init' method of CommandService may not have been called yet. + */ + ['__on_boot.consolidation'] () { + this._register_commands(this.services.get('commands')); + } adapt_id_ (id) { - // let shorten = false; - // // Check if id uses characters that aren't on a US QWERTY keyboard. - // if ( /[^\x20-\x7E]/.test(id) ) shorten = true; - - // // Check if id is too long - // if ( id.length > 20 ) shorten = true; let shorten = true; if ( shorten ) { const rng = seedrandom(id); - id = generate_identifier('-', rng); + id = this.identutil.generate_identifier('-', rng); } return id; } + /** + * Method to create an alarm with the given ID, message, and fields. + * If the ID already exists, it will be updated with the new fields + * and the occurrence count will be incremented. + * + * @param {string} id - Unique identifier for the alarm. + * @param {string} message - Message associated with the alarm. + * @param {object} fields - Additional information about the alarm. + */ create (id, message, fields) { this.log.error(`upcoming alarm: ${id}: ${message}`); let existing = false; + /** + * Method to create an alarm with the given ID, message, and fields. + * If the ID already exists, it will be updated with the new fields. + * @param {string} id - Unique identifier for the alarm. + * @param {string} message - Message associated with the alarm. + * @param {object} fields - Additional information about the alarm. + * @returns {void} + */ const alarm = (() => { const short_id = this.adapt_id_(id); @@ -116,12 +140,34 @@ class AlarmService extends BaseService { }; Object.defineProperty(alarm, 'count', { + /** + * Method to create a new alarm. + * + * This method takes an id, message, and optional fields as parameters. + * It creates a new alarm object with the provided id and message, + * and adds it to the alarms object. It also keeps track of the number of occurrences of the alarm. + * If the alarm already exists, it increments the occurrence count and calls the handle\_alarm\_repeat\_ method. + * If it's a new alarm, it calls the handle\_alarm\_on\_ method. + * + * @param {string} id - The unique identifier for the alarm. + * @param {string} message - The message associated with the alarm. + * @param {object} [fields] - Optional fields associated with the alarm. + * @returns {void} + */ get () { return alarm.timestamps?.length ?? 0; } }); Object.defineProperty(alarm, 'id_string', { + /** + * Method to handle creating a new alarm with given parameters. + * This method adds the alarm to the `alarms` object, updates the occurrences count, + * and processes any known errors that may apply to the alarm. + * @param {string} id - The unique identifier for the alarm. + * @param {string} message - The message associated with the alarm. + * @param {Object} fields - Additional fields to associate with the alarm. + */ get () { if ( alarm.id.length < 20 ) { return alarm.id; @@ -170,6 +216,11 @@ class AlarmService extends BaseService { } } + /** + * Method to clear an alarm with the given ID. + * @param {*} id - The ID of the alarm to clear. + * @returns {void} + */ clear (id) { const alarm = this.alarms[id]; if ( !alarm ) { @@ -252,7 +303,7 @@ class AlarmService extends BaseService { svc_devConsole.add_widget(this.alarm_widget); } - const args = Context.get('args') ?? {}; + const args = this.Context.get('args') ?? {}; if ( args['quit-on-alarm'] ) { const svc_shutdown = this.services.get('shutdown'); svc_shutdown.shutdown({ @@ -284,7 +335,7 @@ class AlarmService extends BaseService { // Write a .log file for the alert that happened try { const lines = []; - lines.push(`ALERT ${alarm.id_string} :: ${alarm.message} (${alarm.count})`), + lines.push(`ALERT ${alarm.id_string} :: ${alarm.message} (${alarm.count})`); lines.push(`started: ${new Date(alarm.started).toISOString()}`); lines.push(`short id: ${alarm.short_id}`); lines.push(`original id: ${alarm.id}`); @@ -296,7 +347,7 @@ class AlarmService extends BaseService { (async () => { try { - await fs.appendFileSync(`alert_${alarm.id}.log`, alert_info + '\n'); + fs.appendFileSync(`alert_${alarm.id}.log`, alert_info + '\n'); } catch (e) { this.log.error(`failed to write alert log: ${e.message}`); } @@ -313,11 +364,32 @@ class AlarmService extends BaseService { ); } + /** + * Method to get an alarm by its ID. + * + * @param {*} id - The ID of the alarm to get. + * @returns + */ get_alarm (id) { return this.alarms[id] ?? this.alarm_aliases[id]; } _register_commands (commands) { + // Function to handle a specific alarm event. + // This comment can be added above line 320. + // This function is responsible for processing specific events related to alarms. + // It can be used for tasks such as updating alarm status, sending notifications, or triggering actions. + // This function is called internally by the AlarmService class. + + // /* + // * handleAlarmEvent - Handles a specific alarm event. + // * + // * @param {Object} alarm - The alarm object containing relevant information. + // * @param {Function} callback - Optional callback function to be called when the event is handled. + // */ + // function handleAlarmEvent(alarm, callback) { + // // Implementation goes here. + // } const completeAlarmID = (args) => { // The alarm ID is the first argument, so return no results if we're on the second or later. if (args.length > 1) @@ -424,7 +496,7 @@ class AlarmService extends BaseService { } log.log(`┏━━ Logs before: ${alarm.id_string} ━━━━`); for ( const lg of occurance.logs ) { - log.log("┃ " + stringify_log_entry(lg)); + log.log("┃ " + this.logutil.stringify_log_entry(lg)); } log.log(`┗━━ Logs before: ${alarm.id_string} ━━━━`); }, diff --git a/src/backend/src/modules/core/ContextService.js b/src/backend/src/modules/core/ContextService.js new file mode 100644 index 0000000000..d8775b8fe3 --- /dev/null +++ b/src/backend/src/modules/core/ContextService.js @@ -0,0 +1,21 @@ +const BaseService = require("../../services/BaseService"); +const { Context } = require("../../util/context"); + +/** + * ContextService provides a way for other services to register a hook to be + * called when a context/subcontext is created. + * + * Contexts are used to provide contextual information in the execution + * context (dynamic scope). They can also be used to identify a "span"; + * a span is a labelled frame of execution that can be used to track + * performance, errors, and other metrics. + */ +class ContextService extends BaseService { + register_context_hook (event, hook) { + Context.context_hooks_[event].push(hook); + } +} + +module.exports = { + ContextService, +}; diff --git a/src/backend/src/modules/core/Core2Module.js b/src/backend/src/modules/core/Core2Module.js new file mode 100644 index 0000000000..d3a79efb25 --- /dev/null +++ b/src/backend/src/modules/core/Core2Module.js @@ -0,0 +1,61 @@ +const { AdvancedBase } = require("@heyputer/putility"); + +/** + * A replacement for CoreModule with as few external relative requires as possible. + * This will eventually be the successor to CoreModule, the main module for Puter's backend. + * + * The scope of this module is: + * - logging and error handling + * - alarm handling + * - services that are tightly coupled with alarm handling are allowed + * - any essential information about server stats or health + * - any very generic service which other services can register + * behavior to. + */ +class Core2Module extends AdvancedBase { + async install (context) { + // === LIBS === // + const useapi = context.get('useapi'); + + const lib = require('./lib/__lib__.js'); + for ( const k in lib ) { + useapi.def(`core.${k}`, lib[k], { assign: true }); + } + + useapi.def('core.context', require('../../util/context.js').Context); + + // === SERVICES === // + const services = context.get('services'); + + const { LogService } = require('./LogService.js'); + services.registerService('log-service', LogService); + + const { AlarmService } = require("./AlarmService.js"); + services.registerService('alarm', AlarmService); + + const { ErrorService } = require("./ErrorService.js"); + services.registerService('error-service', ErrorService); + + const { PagerService } = require("./PagerService.js"); + services.registerService('pager', PagerService); + + const { ExpectationService } = require("./ExpectationService.js"); + services.registerService('expectations', ExpectationService); + + const { ProcessEventService } = require("./ProcessEventService.js"); + services.registerService('process-event', ProcessEventService); + + const { ServerHealthService } = require("./ServerHealthService.js"); + services.registerService('server-health', ServerHealthService); + + const { ParameterService } = require("./ParameterService.js"); + services.registerService('params', ParameterService); + + const { ContextService } = require('./ContextService.js'); + services.registerService('context', ContextService); + } +} + +module.exports = { + Core2Module, +}; diff --git a/src/backend/src/services/runtime-analysis/ErrorService.js b/src/backend/src/modules/core/ErrorService.js similarity index 50% rename from src/backend/src/services/runtime-analysis/ErrorService.js rename to src/backend/src/modules/core/ErrorService.js index e864755c34..cef2064a41 100644 --- a/src/backend/src/services/runtime-analysis/ErrorService.js +++ b/src/backend/src/modules/core/ErrorService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -16,8 +17,20 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -const BaseService = require("../BaseService"); +const BaseService = require("../../services/BaseService"); + +/** +* **ErrorContext Class** +* +* The `ErrorContext` class is designed to encapsulate error reporting functionality within a specific logging context. +* It facilitates the reporting of errors by providing a method to log error details along with additional contextual information. +* +* @class +* @classdesc Provides a context for error reporting with specific logging details. +* @param {ErrorService} error_service - The error service instance to use for reporting errors. +* @param {object} log_context - The logging context to associate with the error reports. +*/ class ErrorContext { constructor (error_service, log_context) { this.error_service = error_service; @@ -32,15 +45,49 @@ class ErrorContext { } } + +/** +* The ErrorService class is responsible for handling and reporting errors within the system. +* It provides methods to initialize the service, create error contexts, and report errors with detailed logging and alarm mechanisms. + +* @class ErrorService +* @extends BaseService +*/ class ErrorService extends BaseService { + /** + * Initializes the ErrorService, setting up the alarm and backup logger services. + * + * @async + * @function init + * @memberof ErrorService + * @returns {Promise} A promise that resolves when the initialization is complete. + */ async init () { const services = this.services; this.alarm = services.get('alarm'); this.backupLogger = services.get('log-service').create('error-service'); } + + /** + * Creates an ErrorContext instance with the provided logging context. + * + * @param {*} log_context The logging context to associate with the error reports. + * @returns {ErrorContext} An ErrorContext instance. + */ create (log_context) { return new ErrorContext(this, log_context); } + + /** + * Reports an error with the specified location and details. + * The "location" is a string up to the callers discretion to identify + * the source of the error. + * + * @param {*} location The location where the error occurred. + * @param {*} fields The error details to report. + * @param {boolean} [alarm=true] Whether to raise an alarm for the error. + * @returns {void} + */ report (location, { source, logger, trace, extra, message }, alarm = true) { message = message ?? source?.message; logger = logger ?? this.backupLogger; diff --git a/src/backend/src/modules/core/ExpectationService.js b/src/backend/src/modules/core/ExpectationService.js new file mode 100644 index 0000000000..b1d8455302 --- /dev/null +++ b/src/backend/src/modules/core/ExpectationService.js @@ -0,0 +1,138 @@ +// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}} +/* + * Copyright (C) 2024 Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +const { v4: uuidv4 } = require('uuid'); +const BaseService = require('../../services/BaseService'); + +/** +* @class ExpectationService +* @extends BaseService +* +* The `ExpectationService` is a specialized service designed to assist in the diagnosis and +* management of errors related to the intricate interactions among asynchronous operations. +* It facilitates tracking and reporting on expectations, enabling better fault isolation +* and resolution in systems where synchronization and timing of operations are crucial. +* +* This service inherits from the `BaseService` and provides methods for registering, +* purging, and handling expectations, making it a valuable tool for diagnosing complex +* runtime behaviors in a system. +*/ +class ExpectationService extends BaseService { + static USE = { + expect: 'core.expect' + }; + + /** + * Constructs the ExpectationService and initializes its internal state. + * This method is intended to be called asynchronously. + * It sets up the `expectations_` array which will be used to track expectations. + * + * @async + */ + async _construct () { + this.expectations_ = []; + } + + /** + * ExpectationService registers its commands at the consolidation phase because + * the '_init' method of CommandService may not have been called yet. + */ + ['__on_boot.consolidation'] () { + const commands = this.services.get('commands'); + commands.registerCommands('expectations', [ + { + id: 'pending', + description: 'lists pending expectations', + handler: async (args, log) => { + this.purgeExpectations_(); + if ( this.expectations_.length < 1 ) { + log.log(`there are none`); + return; + } + for ( const expectation of this.expectations_ ) { + expectation.report(log); + } + } + } + ]); + } + + /** + * Initializes the ExpectationService, setting up interval functions and registering commands. + * + * This method sets up a periodic interval to purge expectations and registers a command + * to list pending expectations. The interval invokes `purgeExpectations_` every second. + * The command 'pending' allows users to list and log all pending expectations. + * + * @returns {Promise} A promise that resolves when initialization is complete. + */ + async _init () { + // TODO: service to track all interval functions? + /** + * Initializes the service by setting up interval functions and registering commands. + * This method sets up a periodic interval function to purge expectations and registers + * a command to list pending expectations. + * + * @returns {void} + */ + + // The comment should be placed above the method at line 68 + setInterval(() => { + this.purgeExpectations_(); + }, 1000); + } + + + /** + * Purges expectations that have been met. + * + * This method iterates through the list of expectations and removes + * those that have been satisfied. Currently, this functionality is + * disabled and needs to be re-enabled. + * + * @returns {void} This method does not return anything. + */ + purgeExpectations_ () { + return; + // TODO: Re-enable this + // for ( let i=0 ; i < this.expectations_.length ; i++ ) { + // if ( this.expectations_[i].check() ) { + // this.expectations_[i] = null; + // } + // } + // this.expectations_ = this.expectations_.filter(v => v !== null); + } + + /** + * Registers an expectation to be tracked by the service. + * + * @param {Object} workUnit - The work unit to track + * @param {string} checkpoint - The checkpoint to expect + * @returns {void} + */ + expect_eventually ({ workUnit, checkpoint }) { + this.expectations_.push(new this.expect.CheckpointExpectation(workUnit, checkpoint)); + } +} + + + +module.exports = { + ExpectationService +}; \ No newline at end of file diff --git a/src/backend/src/services/runtime-analysis/LogService.js b/src/backend/src/modules/core/LogService.js similarity index 73% rename from src/backend/src/services/runtime-analysis/LogService.js rename to src/backend/src/modules/core/LogService.js index 5daec8a952..e91076c18d 100644 --- a/src/backend/src/services/runtime-analysis/LogService.js +++ b/src/backend/src/modules/core/LogService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"xai"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -27,7 +28,8 @@ const LOG_LEVEL_SYSTEM = logSeverity(4, 'SYSTEM', '33;1', 'system'); const winston = require('winston'); const { Context } = require('../../util/context'); -const BaseService = require('../BaseService'); +const BaseService = require('../../services/BaseService'); +const { stringify_log_entry } = require('./lib/log'); require('winston-daily-rotate-file'); const WINSTON_LEVELS = { @@ -41,6 +43,13 @@ const WINSTON_LEVELS = { silly: 60 }; + +/** +* Represents a logging context within the LogService. +* This class is used to manage logging operations with specific context information, +* allowing for hierarchical logging structures and dynamic field additions. +* @class LogContext +*/ class LogContext { constructor (logService, { crumbs, fields }) { this.logService = logService; @@ -96,8 +105,12 @@ class LogContext { ); } - // convenience method to get a trace id that isn't as difficult - // for a human to read as a uuid. + /** + * Generates a human-readable trace ID for logging purposes. + * + * @returns {string} A trace ID in the format 'xxxxxx-xxxxxx' where each segment is a + * random string of six lowercase letters and digits. + */ mkid () { // generate trace id const trace_id = []; @@ -107,55 +120,40 @@ class LogContext { return trace_id.join('-'); } - // add a trace id to this logging context + /** + * Adds a trace id to this logging context for tracking purposes. + * @returns {LogContext} The current logging context with the trace id added. + */ traceOn () { this.fields.trace_id = this.mkid(); return this; } + + /** + * Gets the log buffer maintained by the LogService. This shows the most + * recent log entries. + * @returns {Array} An array of log entries stored in the buffer. + */ get_log_buffer () { return this.logService.get_log_buffer(); } } -let log_epoch = Date.now(); -const stringify_log_entry = ({ prefix, log_lvl, crumbs, message, fields, objects }) => { - const { colorize } = require('json-colorizer'); - - let lines = [], m; - const lf = () => { - if ( ! m ) return; - lines.push(m); - m = ''; - } - - m = prefix ? `${prefix} ` : ''; - m += `\x1B[${log_lvl.esc}m[${log_lvl.label}\x1B[0m`; - for ( const crumb of crumbs ) { - m += `::${crumb}`; - } - m += `\x1B[${log_lvl.esc}m]\x1B[0m`; - if ( fields.timestamp ) { - // display seconds since logger epoch - const n = (fields.timestamp - log_epoch) / 1000; - m += ` (${n.toFixed(3)}s)`; - } - m += ` ${message} `; - lf(); - for ( const k in fields ) { - if ( k === 'timestamp' ) continue; - let v; try { - v = colorize(JSON.stringify(fields[k])); - } catch (e) { - v = '' + fields[k]; - } - m += ` \x1B[1m${k}:\x1B[0m ${v}`; - lf(); - } - return lines.join('\n'); -}; - - +/** +* Timestamp in milliseconds since the epoch, used for calculating log entry duration. +*/ + +/** +* @class DevLogger +* @classdesc +* A development logger class designed for logging messages during development. +* This logger can either log directly to console or delegate logging to another logger. +* It provides functionality to turn logging on/off, and can optionally write logs to a file. +* +* @param {function} log - The logging function, typically `console.log` or similar. +* @param {object} [opt_delegate] - An optional logger to which log messages can be delegated. +*/ class DevLogger { // TODO: this should eventually delegate to winston logger constructor (log, opt_delegate) { @@ -195,6 +193,13 @@ class DevLogger { } } + +/** +* @class NullLogger +* @description A logger that does nothing, effectively disabling logging. +* This class is used when logging is not desired or during development +* to avoid performance overhead or for testing purposes. +*/ class NullLogger { // TODO: this should eventually delegate to winston logger constructor (log, opt_delegate) { @@ -208,6 +213,12 @@ class NullLogger { } } + +/** +* WinstonLogger Class +* +* A logger that delegates log messages to a Winston logger instance. +*/ class WinstonLogger { constructor (winst) { this.winst = winst; @@ -222,6 +233,15 @@ class WinstonLogger { } } + +/** +* @class TimestampLogger +* @classdesc A logger that adds timestamps to log messages before delegating them to another logger. +* This class wraps another logger instance to ensure that all log messages include a timestamp, +* which can be useful for tracking the sequence of events in a system. +* +* @param {Object} delegate - The logger instance to which the timestamped log messages are forwarded. +*/ class TimestampLogger { constructor (delegate) { this.delegate = delegate; @@ -232,6 +252,15 @@ class TimestampLogger { } } + +/** +* The `BufferLogger` class extends the logging functionality by maintaining a buffer of log entries. +* This class is designed to: +* - Store a specified number of recent log messages. +* - Allow for retrieval of these logs for debugging or monitoring purposes. +* - Ensure that the log buffer does not exceed the defined size by removing older entries when necessary. +* - Delegate logging messages to another logger while managing its own buffer. +*/ class BufferLogger { constructor (size, delegate) { this.size = size; @@ -247,6 +276,14 @@ class BufferLogger { } } + +/** +* Represents a custom logger that can modify log messages before they are passed to another logger. +* @class CustomLogger +* @extends {Object} +* @param {Object} delegate - The delegate logger to which modified log messages will be passed. +* @param {Function} callback - A callback function that modifies log parameters before delegation. +*/ class CustomLogger { constructor (delegate, callback) { this.delegate = delegate; @@ -279,17 +316,39 @@ class CustomLogger { } } + +/** +* The `LogService` class extends `BaseService` and is responsible for managing and +* orchestrating various logging functionalities within the application. It handles +* log initialization, middleware registration, log directory management, and +* provides methods for creating log contexts and managing log output levels. +*/ class LogService extends BaseService { static MODULES = { path: require('path'), } + /** + * Defines the modules required by the LogService class. + * This static property contains modules that are used for file path operations. + * @property {Object} MODULES - An object containing required modules. + * @property {Object} MODULES.path - The Node.js path module for handling and resolving file paths. + */ async _construct () { this.loggers = []; this.bufferLogger = null; } + + /** + * Registers a custom logging middleware with the LogService. + * @param {*} callback - The callback function that modifies log parameters before delegation. + */ register_log_middleware (callback) { this.loggers[0] = new CustomLogger(this.loggers[0], callback); } + + /** + * Registers logging commands with the command service. + */ ['__on_boot.consolidation'] () { const commands = this.services.get('commands'); commands.registerCommands('logs', [ @@ -332,6 +391,15 @@ class LogService extends BaseService { } ]); } + /** + * Registers logging commands with the command service. + * + * This method sets up various logging commands that can be used to + * interact with the log output, such as toggling log display, + * starting/stopping log recording, and toggling log indentation. + * + * @memberof LogService + */ async _init () { const config = this.global_config; @@ -423,6 +491,13 @@ class LogService extends BaseService { globalThis.root_context.set('logger', this.create('root-context')); } + /** + * Create a new log context with the specified prefix + * + * @param {1} prefix - The prefix for the log context + * @param {*} fields - Optional fields to include in the log context + * @returns {LogContext} A new log context with the specified prefix and fields + */ create (prefix, fields = {}) { const logContext = new LogContext( this, @@ -457,6 +532,15 @@ class LogService extends BaseService { } } + + /** + * Ensures that a log directory exists for logging purposes. + * This method attempts to create or locate a directory for log files, + * falling back through several predefined paths if the preferred + * directory does not exist or cannot be created. + * + * @throws {Error} If no suitable log directory can be found or created. + */ ensure_log_directory_ () { // STEP 1: Try /var/puter/logs/heyputer { @@ -506,12 +590,24 @@ class LogService extends BaseService { throw new Error('Unable to create or find log directory'); } + /** + * Generates a sanitized file path for log files. + * + * @param {string} name - The name of the log file, which will be sanitized to remove any path characters. + * @returns {string} A sanitized file path within the log directory. + */ get_log_file (name) { // sanitize name: cannot contain path characters name = name.replace(/[^a-zA-Z0-9-_]/g, '_'); return this.modules.path.join(this.log_directory, name); } + + /** + * Get the most recent log entries from the buffer maintained by the LogService. + * By default, the buffer contains the last 20 log entries. + * @returns + */ get_log_buffer () { return this.bufferLogger.buffer; } diff --git a/src/backend/src/services/runtime-analysis/PagerService.js b/src/backend/src/modules/core/PagerService.js similarity index 67% rename from src/backend/src/services/runtime-analysis/PagerService.js rename to src/backend/src/modules/core/PagerService.js index f68c5d97e1..a5bad19f1e 100644 --- a/src/backend/src/services/runtime-analysis/PagerService.js +++ b/src/backend/src/modules/core/PagerService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -17,19 +18,47 @@ * along with this program. If not, see . */ const pdjs = require('@pagerduty/pdjs'); -const BaseService = require('../BaseService'); +const BaseService = require('../../services/BaseService'); const util = require('util'); -const { Context } = require('../../util/context'); + +/** +* @class PagerService +* @extends BaseService +* @description The PagerService class is responsible for handling pager alerts. +* It extends the BaseService class and provides methods for constructing, +* initializing, and managing alert handlers. The class interacts with PagerDuty +* through the pdjs library to send alerts and integrates with other services via +* command registration. +*/ class PagerService extends BaseService { + static USE = { + Context: 'core.context', + } + async _construct () { this.config = this.global_config.pager; this.alertHandlers_ = []; } - async _init () { - const services = this.services; + + /** + * PagerService registers its commands at the consolidation phase because + * the '_init' method of CommandService may not have been called yet. + */ + ['__on_boot.consolidation'] () { + this._register_commands(this.services.get('commands')); + } + /** + * Initializes the PagerService instance by setting the configuration and + * initializing an empty alert handler array. + * + * @async + * @memberOf PagerService + * @returns {Promise} + */ + async _init () { this.alertHandlers_ = []; if ( ! this.config ) { @@ -37,10 +66,15 @@ class PagerService extends BaseService { } this.onInit(); - - this._register_commands(services.get('commands')); } + /** + * Initializes PagerDuty configuration and registers alert handlers. + * If PagerDuty is enabled in the configuration, it sets up an alert handler + * to send alerts to PagerDuty. + * + * @method onInit + */ onInit () { if ( this.config.pagerduty && this.config.pagerduty.enabled ) { this.alertHandlers_.push(async alert => { @@ -56,7 +90,7 @@ class PagerService extends BaseService { server_id: this.global_config.server_id, }; - const ctx = Context.get(undefined, { allow_fallback: true }); + const ctx = this.Context.get(undefined, { allow_fallback: true }); // Add request payload if any exists const req = ctx.get('req'); @@ -89,6 +123,15 @@ class PagerService extends BaseService { } } + + /** + * Sends an alert to all registered alert handlers. + * + * This method iterates through all alert handlers and attempts to send the alert. + * If any handler fails to send the alert, an error message is logged. + * + * @param {Object} alert - The alert object containing details about the alert. + */ async alert (alert) { for ( const handler of this.alertHandlers_ ) { try { diff --git a/src/backend/src/services/ParameterService.js b/src/backend/src/modules/core/ParameterService.js similarity index 69% rename from src/backend/src/services/ParameterService.js rename to src/backend/src/modules/core/ParameterService.js index 8ff72367df..25c4ea2313 100644 --- a/src/backend/src/services/ParameterService.js +++ b/src/backend/src/modules/core/ParameterService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"claude"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -17,32 +18,37 @@ * along with this program. If not, see . */ -const BaseService = require("./BaseService"); +const BaseService = require("../../services/BaseService"); -/* - * Copyright (C) 2024 Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ +/** +* @class ParameterService +* @extends BaseService +* @description Service class for managing system parameters and their values. +* Provides functionality for creating, getting, setting, and subscribing to parameters. +* Supports parameter binding to instances and includes command registration for parameter management. +* Parameters can have constraints, default values, and change listeners. +*/ class ParameterService extends BaseService { + /** + * Parameter service for managing system-wide parameters + * @extends BaseService + * @class + * @description Handles registration, storage, and access of parameters across services. + * Parameters can be bound to instances, subscribed to for changes, and accessed via commands. + * Each parameter has a unique service-scoped ID and optional constraints. + */ _construct () { this.parameters_ = []; } - _init () { + + /** + * Initializes the service by registering commands with the command service. + * This method is called during service startup to set up command handlers + * for parameter management. + * @private + */ + ['__on_boot.consolidation'] () { this._registerCommands(this.services.get('commands')); } @@ -63,6 +69,13 @@ class ParameterService extends BaseService { } } + + /** + * Gets the value of a parameter by its ID + * @param {string} id - The unique identifier of the parameter to retrieve + * @returns {Promise<*>} The current value of the parameter + * @throws {Error} If parameter with given ID is not found + */ async get(id) { const parameter = this._get_param(id); return await parameter.get(); @@ -87,6 +100,10 @@ class ParameterService extends BaseService { } _registerCommands (commands) { + /** + * Registers parameter-related commands with the command service + * @param {Object} commands - The command service instance to register with + */ const completeParameterName = (args) => { // The parameter name is the first argument, so return no results if we're on the second or later. if (args.length > 1) @@ -146,6 +163,13 @@ class ParameterService extends BaseService { } } + +/** +* @class Parameter +* @description Represents a configurable parameter with value management, constraints, and change notification capabilities. +* Provides functionality for setting/getting values, binding to object instances, and subscribing to value changes. +* Supports validation through configurable constraints and maintains a list of value change listeners. +*/ class Parameter { constructor(spec) { this.spec_ = spec; @@ -156,6 +180,14 @@ class Parameter { } } + + /** + * Sets a new value for the parameter after validating against constraints + * @param {*} value - The new value to set for the parameter + * @throws {Error} If the value fails any constraint checks + * @fires valueListeners with new value and old value + * @async + */ async set (value) { for ( const constraint of (this.spec_.constraints ?? []) ) { if ( ! await constraint.check(value) ) { @@ -170,6 +202,11 @@ class Parameter { } } + + /** + * Gets the current value of this parameter + * @returns {Promise<*>} The parameter's current value + */ async get () { return this.value_; } diff --git a/src/backend/src/services/runtime-analysis/ProcessEventService.js b/src/backend/src/modules/core/ProcessEventService.js similarity index 52% rename from src/backend/src/services/runtime-analysis/ProcessEventService.js rename to src/backend/src/modules/core/ProcessEventService.js index 30e73fc09c..e994bdcc8e 100644 --- a/src/backend/src/services/runtime-analysis/ProcessEventService.js +++ b/src/backend/src/modules/core/ProcessEventService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"claude"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -16,18 +17,36 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -const { Context } = require("../../util/context"); -class ProcessEventService { - constructor ({ services }) { +const BaseService = require("../../services/BaseService"); + +/** +* Service class that handles process-wide events and errors. +* Provides centralized error handling for uncaught exceptions and unhandled promise rejections. +* Sets up event listeners on the process object to capture and report critical errors +* through the logging and error reporting services. +* +* @class ProcessEventService +*/ +class ProcessEventService extends BaseService { + static USE = { + Context: 'core.context', + }; + + _init () { + const services = this.services; const log = services.get('log-service').create('process-event-service'); const errors = services.get('error-service').create(log); - // TODO: when the service lifecycle is implemented, but these - // in the init hook - process.on('uncaughtException', async (err, origin) => { - await Context.allow_fallback(async () => { + /** + * Handles uncaught exceptions in the process + * Sets up an event listener that reports errors when uncaught exceptions occur + * @param {Error} err - The uncaught exception error object + * @param {string} origin - The origin of the uncaught exception + * @returns {Promise} + */ + await this.Context.allow_fallback(async () => { errors.report('process:uncaughtException', { source: err, origin, @@ -39,7 +58,13 @@ class ProcessEventService { }); process.on('unhandledRejection', async (reason, promise) => { - await Context.allow_fallback(async () => { + /** + * Handles unhandled promise rejections by reporting them to the error service + * @param {*} reason - The rejection reason/error + * @param {Promise} promise - The rejected promise + * @returns {Promise} Resolves when error is reported + */ + await this.Context.allow_fallback(async () => { errors.report('process:unhandledRejection', { source: reason, promise, diff --git a/src/backend/src/modules/core/README.md b/src/backend/src/modules/core/README.md new file mode 100644 index 0000000000..c55b3841c5 --- /dev/null +++ b/src/backend/src/modules/core/README.md @@ -0,0 +1,269 @@ +# Core2Module + +A replacement for CoreModule with as few external relative requires as possible. +This will eventually be the successor to CoreModule, the main module for Puter's backend. + +## Services + +### AlarmService + +AlarmService class is responsible for managing alarms. +It provides methods for creating, clearing, and handling alarms. + +#### Listeners + +##### `boot.consolidation` + +AlarmService registers its commands at the consolidation phase because +the '_init' method of CommandService may not have been called yet. + +#### Methods + +##### `create` + +Method to create an alarm with the given ID, message, and fields. +If the ID already exists, it will be updated with the new fields +and the occurrence count will be incremented. + +###### Parameters + +- **id:** Unique identifier for the alarm. +- **message:** Message associated with the alarm. +- **fields:** Additional information about the alarm. + +##### `clear` + +Method to clear an alarm with the given ID. + +###### Parameters + +- **id:** The ID of the alarm to clear. + +##### `get_alarm` + +Method to get an alarm by its ID. + +###### Parameters + +- **id:** The ID of the alarm to get. + +### ErrorService + +The ErrorService class is responsible for handling and reporting errors within the system. +It provides methods to initialize the service, create error contexts, and report errors with detailed logging and alarm mechanisms. + +#### Methods + +##### `init` + +Initializes the ErrorService, setting up the alarm and backup logger services. + +##### `create` + +Creates an ErrorContext instance with the provided logging context. + +###### Parameters + +- **log_context:** The logging context to associate with the error reports. + +##### `report` + +Reports an error with the specified location and details. +The "location" is a string up to the callers discretion to identify +the source of the error. + +###### Parameters + +- **location:** The location where the error occurred. +- **fields:** The error details to report. + +### ExpectationService + + + +#### Listeners + +##### `boot.consolidation` + +ExpectationService registers its commands at the consolidation phase because +the '_init' method of CommandService may not have been called yet. + +#### Methods + +##### `expect_eventually` + +Registers an expectation to be tracked by the service. + +###### Parameters + +- **workUnit:** The work unit to track +- **checkpoint:** The checkpoint to expect + +### LogService + +The `LogService` class extends `BaseService` and is responsible for managing and +orchestrating various logging functionalities within the application. It handles +log initialization, middleware registration, log directory management, and +provides methods for creating log contexts and managing log output levels. + +#### Listeners + +##### `boot.consolidation` + +Registers logging commands with the command service. + +#### Methods + +##### `register_log_middleware` + +Registers a custom logging middleware with the LogService. + +###### Parameters + +- **callback:** The callback function that modifies log parameters before delegation. + +##### `create` + +Create a new log context with the specified prefix + +###### Parameters + +- **prefix:** The prefix for the log context +- **fields:** Optional fields to include in the log context + +##### `get_log_file` + +Generates a sanitized file path for log files. + +###### Parameters + +- **name:** The name of the log file, which will be sanitized to remove any path characters. + +##### `get_log_buffer` + +Get the most recent log entries from the buffer maintained by the LogService. +By default, the buffer contains the last 20 log entries. + +### PagerService + + + +#### Listeners + +##### `boot.consolidation` + +PagerService registers its commands at the consolidation phase because +the '_init' method of CommandService may not have been called yet. + +#### Methods + +##### `onInit` + +Initializes PagerDuty configuration and registers alert handlers. +If PagerDuty is enabled in the configuration, it sets up an alert handler +to send alerts to PagerDuty. + +##### `alert` + +Sends an alert to all registered alert handlers. + +This method iterates through all alert handlers and attempts to send the alert. +If any handler fails to send the alert, an error message is logged. + +###### Parameters + +- **alert:** The alert object containing details about the alert. + +### ProcessEventService + +Service class that handles process-wide events and errors. +Provides centralized error handling for uncaught exceptions and unhandled promise rejections. +Sets up event listeners on the process object to capture and report critical errors +through the logging and error reporting services. + +## Libraries + +### core.expect + +### core.util.identutil + +#### Functions + +##### `randomItem` + +Select a random item from an array using a random number generator function. + +###### Parameters + +- **arr:** The array to select an item from + +### core.util.logutil + +#### Functions + +##### `stringify_log_entry` + +Stringifies a log entry into a formatted string for console output. + +###### Parameters + +- **logEntry:** The log entry object containing: + +### stdio + +#### Functions + +##### `visible_length` + +METADATA // {"ai-commented":{"service":"claude"}} + +##### `split_lines` + +Split a string into lines according to the terminal width, +preserving ANSI escape sequences, and return an array of lines. + +###### Parameters + +- **str:** The string to split into lines + +### core.util.strutil + +#### Functions + +##### `quot` + +METADATA // {"def":"core.util.strutil","ai-params":{"service":"claude"},"ai-commented":{"service":"claude"}} + +##### `osclink` + +Creates an OSC 8 hyperlink sequence for terminal output + +###### Parameters + +- **url:** The URL to link to + +##### `format_as_usd` + +Formats a number as a USD currency string with appropriate decimal places + +###### Parameters + +- **amount:** The amount to format + +## Notes + +### Outside Imports + +This module has external relative imports. When these are +removed it may become possible to move this module to an +extension. + +**Imports:** +- `../../services/BaseService.js` +- `../../util/context.js` +- `../../services/BaseService` (use.BaseService) +- `../../services/BaseService` (use.BaseService) +- `../../util/context` +- `../../services/BaseService` (use.BaseService) +- `../../services/BaseService` (use.BaseService) +- `../../services/BaseService` (use.BaseService) diff --git a/src/backend/src/services/runtime-analysis/ServerHealthService.js b/src/backend/src/modules/core/ServerHealthService.js similarity index 57% rename from src/backend/src/services/runtime-analysis/ServerHealthService.js rename to src/backend/src/modules/core/ServerHealthService.js index 702231b08c..f5bdfdcec7 100644 --- a/src/backend/src/services/runtime-analysis/ServerHealthService.js +++ b/src/backend/src/modules/core/ServerHealthService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"xai"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -16,19 +17,45 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -const BaseService = require("../BaseService"); -const { SECOND } = require("../../util/time"); -const { parse_meminfo } = require("../../util/linux"); -const { asyncSafeSetInterval, TeePromise } = require("../../util/promise"); - +const BaseService = require("../../services/BaseService"); +const { time, promise } = require("@heyputer/putility").libs; + + +/** +* The ServerHealthService class provides comprehensive health monitoring for the server. +* It extends the BaseService class to include functionality for: +* - Periodic system checks (e.g., RAM usage, service checks) +* - Managing health check results and failures +* - Triggering alarms for critical conditions +* - Logging and managing statistics for health metrics +* +* This service is designed to work primarily on Linux systems, reading system metrics +* from `/proc/meminfo` and handling alarms via an external 'alarm' service. +*/ class ServerHealthService extends BaseService { + static USE = { + linuxutil: 'core.util.linuxutil' + }; + static MODULES = { fs: require('fs'), } + /** + * Defines the modules used by ServerHealthService. + * This static property is used to initialize and access system modules required for health checks. + * @type {Object} + * @property {fs} fs - The file system module for reading system information. + */ _construct () { this.checks_ = []; this.failures_ = []; } + /** + * Initializes the internal checks and failure tracking for the service. + * This method sets up empty arrays to store health checks and their failure statuses. + * + * @private + */ async _init () { this.init_service_checks_(); @@ -45,7 +72,6 @@ class ServerHealthService extends BaseService { */ - const min_free_KiB = 1024 * 1024; // 1 GiB const min_available_KiB = 1024 * 1024 * 2; // 2 GiB const svc_alarm = this.services.get('alarm'); @@ -59,11 +85,19 @@ class ServerHealthService extends BaseService { if ( this.config.no_system_checks ) return; + + /** + * Adds a health check to the service. + * + * @param {string} name - The name of the health check. + * @param {Function} fn - The function to execute for the health check. + * @returns {Object} A chainable object to add failure handlers. + */ this.add_check('ram-usage', async () => { const meminfo_text = await this.modules.fs.promises.readFile( '/proc/meminfo', 'utf8' ); - const meminfo = parse_meminfo(meminfo_text); + const meminfo = this.linuxutil.parse_meminfo(meminfo_text); const alarm_fields = { mem_free: meminfo.MemFree, mem_available: meminfo.MemAvailable, @@ -78,16 +112,43 @@ class ServerHealthService extends BaseService { }); } + + /** + * Initializes service health checks by setting up periodic checks. + * This method configures an interval-based execution of health checks, + * handles timeouts, and manages failure states. + * + * @param {none} - This method does not take any parameters. + * @returns {void} - This method does not return any value. + */ init_service_checks_ () { const svc_alarm = this.services.get('alarm'); - asyncSafeSetInterval(async () => { + /** + * Initializes periodic health checks for the server. + * + * This method sets up an interval to run all registered health checks + * at a specified frequency. It manages the execution of checks, handles + * timeouts, and logs errors or triggers alarms when checks fail. + * + * @private + * @method init_service_checks_ + * @memberof ServerHealthService + * @param {none} - No parameters are passed to this method. + * @returns {void} + */ + promise.asyncSafeSetInterval(async () => { this.log.tick('service checks'); const check_failures = []; for ( const { name, fn, chainable } of this.checks_ ) { - const p_timeout = new TeePromise(); + const p_timeout = new promise.TeePromise(); + /** + * Creates a TeePromise to handle potential timeouts during health checks. + * + * @returns {Promise} A promise that can be resolved or rejected from multiple places. + */ const timeout = setTimeout(() => { p_timeout.reject(new Error('Health check timed out')); - }, 5 * SECOND); + }, 5 * time.SECOND); try { await Promise.race([ fn(), @@ -120,7 +181,7 @@ class ServerHealthService extends BaseService { } this.failures_ = check_failures; - }, 10 * SECOND, null, { + }, 10 * time.SECOND, null, { onBehindSchedule: (drift) => { svc_alarm.create( 'health-checks-behind-schedule', @@ -131,6 +192,14 @@ class ServerHealthService extends BaseService { }); } + + /** + * Retrieves the current server health statistics. + * + * @returns {Object} An object containing the current health statistics. + * This method returns a shallow copy of the internal `stats_` object to prevent + * direct manipulation of the service's data. + */ async get_stats () { return { ...this.stats_ }; } @@ -147,6 +216,14 @@ class ServerHealthService extends BaseService { return chainable; } + + /** + * Retrieves the current health status of the server. + * + * @returns {Object} An object containing: + * - `ok` {boolean}: Indicates if all health checks passed. + * - `failed` {Array}: An array of names of failed health checks, if any. + */ get_status () { const failures = this.failures_.map(v => v.name); return { diff --git a/src/backend/src/modules/core/lib/__lib__.js b/src/backend/src/modules/core/lib/__lib__.js new file mode 100644 index 0000000000..25d503ea7f --- /dev/null +++ b/src/backend/src/modules/core/lib/__lib__.js @@ -0,0 +1,9 @@ +module.exports = { + util: { + logutil: require('./log.js'), + identutil: require('./identifier.js'), + stdioutil: require('./stdio.js'), + linuxutil: require('./linux.js'), + }, + expect: require('./expect.js'), +}; diff --git a/src/backend/src/modules/core/lib/expect.js b/src/backend/src/modules/core/lib/expect.js new file mode 100644 index 0000000000..1b80aa78b8 --- /dev/null +++ b/src/backend/src/modules/core/lib/expect.js @@ -0,0 +1,74 @@ +// METADATA // {"def":"core.expect"} +const { v4: uuidv4 } = require('uuid'); + +/** +* @class WorkUnit +* @description The WorkUnit class represents a unit of work that can be tracked and monitored for checkpoints. +* It includes methods to create instances, set checkpoints, and manage the state of the work unit. +*/ +class WorkUnit { + /** + * Represents a unit of work with checkpointing capabilities. + * + * @class + */ + + /** + * Creates and returns a new instance of WorkUnit. + * + * @static + * @returns {WorkUnit} A new instance of WorkUnit. + */ + static create () { + return new WorkUnit(); + } + /** + * Creates a new instance of the WorkUnit class. + * @static + * @returns {WorkUnit} A new WorkUnit instance. + */ + constructor () { + this.id = uuidv4(); + this.checkpoint_ = null; + } + checkpoint (label) { + console.log('CHECKPOINT', label); + this.checkpoint_ = label; + } +} + +/** +* @class CheckpointExpectation +* @classdesc The CheckpointExpectation class is used to represent an expectation that a specific checkpoint +* will be reached during the execution of a work unit. It includes methods to check if the checkpoint has +* been reached and to report the results of this check. +*/ +class CheckpointExpectation { + constructor (workUnit, checkpoint) { + this.workUnit = workUnit; + this.checkpoint = checkpoint; + } + /** + * Constructor for CheckpointExpectation class. + * Initializes the instance with a WorkUnit and a checkpoint label. + * @param {WorkUnit} workUnit - The work unit associated with the checkpoint. + * @param {string} checkpoint - The checkpoint label to be checked. + */ + check () { + // TODO: should be true if checkpoint was ever reached + return this.workUnit.checkpoint_ == this.checkpoint; + } + report (log) { + if ( this.check() ) return; + log.log( + `operation(${this.workUnit.id}): ` + + `expected ${JSON.stringify(this.checkpoint)} ` + + `and got ${JSON.stringify(this.workUnit.checkpoint_)}.` + ); + } +} + +module.exports = { + WorkUnit, + CheckpointExpectation, +}; diff --git a/src/backend/src/modules/core/lib/identifier.js b/src/backend/src/modules/core/lib/identifier.js new file mode 100644 index 0000000000..1e5e13718c --- /dev/null +++ b/src/backend/src/modules/core/lib/identifier.js @@ -0,0 +1,128 @@ +// METADATA // {"def":"core.util.identutil","ai-commented":{"service":"claude"}} +/* + * Copyright (C) 2024 Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +const adjectives = [ + 'amazing', 'ambitious', 'articulate', 'cool', 'bubbly', 'mindful', 'noble', 'savvy', 'serene', + 'sincere', 'sleek', 'sparkling', 'spectacular', 'splendid', 'spotless', 'stunning', + 'awesome', 'beaming', 'bold', 'brilliant', 'cheerful', 'modest', 'motivated', + 'friendly', 'fun', 'funny', 'generous', 'gifted', 'graceful', 'grateful', + 'passionate', 'patient', 'peaceful', 'perceptive', 'persistent', + 'helpful', 'sensible', 'loyal', 'honest', 'clever', 'capable', + 'calm', 'smart', 'genius', 'bright', 'charming', 'creative', 'diligent', 'elegant', 'fancy', + 'colorful', 'avid', 'active', 'gentle', 'happy', 'intelligent', + 'jolly', 'kind', 'lively', 'merry', 'nice', 'optimistic', 'polite', + 'quiet', 'relaxed', 'silly', 'witty', 'young', + 'strong', 'brave', 'agile', 'bold', 'confident', 'daring', + 'fearless', 'heroic', 'mighty', 'powerful', 'valiant', 'wise', 'wonderful', 'zealous', + 'warm', 'swift', 'neat', 'tidy', 'nifty', 'lucky', 'keen', + 'blue', 'red', 'aqua', 'green', 'orange', 'pink', 'purple', 'cyan', 'magenta', 'lime', + 'teal', 'lavender', 'beige', 'maroon', 'navy', 'olive', 'silver', 'gold', 'ivory', +]; + +const nouns = [ + 'street', 'roof', 'floor', 'tv', 'idea', 'morning', 'game', 'wheel', 'bag', 'clock', 'pencil', 'pen', + 'magnet', 'chair', 'table', 'house', 'room', 'book', 'car', 'tree', 'candle', 'light', 'planet', + 'flower', 'bird', 'fish', 'sun', 'moon', 'star', 'cloud', 'rain', 'snow', 'wind', 'mountain', + 'river', 'lake', 'sea', 'ocean', 'island', 'bridge', 'road', 'train', 'plane', 'ship', 'bicycle', + 'circle', 'square', 'garden', 'harp', 'grass', 'forest', 'rock', 'cake', 'pie', 'cookie', 'candy', + 'butterfly', 'computer', 'phone', 'keyboard', 'mouse', 'cup', 'plate', 'glass', 'door', + 'window', 'key', 'wallet', 'pillow', 'bed', 'blanket', 'soap', 'towel', 'lamp', 'mirror', + 'camera', 'hat', 'shirt', 'pants', 'shoes', 'watch', 'ring', + 'necklace', 'ball', 'toy', 'doll', 'kite', 'balloon', 'guitar', 'violin', 'piano', 'drum', + 'trumpet', 'flute', 'viola', 'cello', 'harp', 'banjo', 'tuba', +] + +const words = { + adjectives, + nouns, +}; + +/** + * Select a random item from an array using a random number generator function. + * + * @param {Array} arr - The array to select an item from + * @param {function} [random=Math.random] - Random number generator function + * @returns {T} A random item from the array + */ +const randomItem = (arr, random) => arr[Math.floor((random ?? Math.random)() * arr.length)]; + +/** + * A function that generates a unique identifier by combining a random adjective, a random noun, and a random number (between 0 and 9999). + * The result is returned as a string with components separated by the specified separator. + * It is useful when you need to create unique identifiers that are also human-friendly. + * + * @param {string} [separator='_'] - The character used to separate the adjective, noun, and number. Defaults to '_' if not provided. + * @param {function} [rng=Math.random] - Random number generator function + * @returns {string} A unique, human-friendly identifier. + * + * @example + * + * let identifier = window.generate_identifier(); + * // identifier would be something like 'clever-idea-123' + * + */ +function generate_identifier(separator = '_', rng = Math.random){ + // return a random combination of first_adj + noun + number (between 0 and 9999) + // e.g. clever-idea-123 + return [ + randomItem(adjectives, rng), + randomItem(nouns, rng), + Math.floor(rng() * 10000), + ].join(separator); +} + +// Character set used for generating human-readable, case-insensitive random codes +const HUMAN_READABLE_CASE_INSENSITIVE = 'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'; + +function generate_random_code(n, { + rng = Math.random, + chars = HUMAN_READABLE_CASE_INSENSITIVE +} = {}) { + let code = ''; + for ( let i = 0 ; i < n ; i++ ) { + code += randomItem(chars, rng); + } + return code; +} + +/** +* Composes a code by combining a mask string with a base-36 converted number +* @param {string} mask - Initial string template to use as base +* @param {number} value - Number to convert to base-36 and append to the right +* @returns {string} Combined uppercase code +*/ +function compose_code(mask, value) { + const right_str = value.toString(36); + let out_str = mask; + console.log('right_str', right_str); + console.log('out_str', out_str); + for ( let i = 0 ; i < right_str.length ; i++ ) { + out_str[out_str.length - 1 - i] = right_str[right_str.length - 1 - i]; + } + + out_str = out_str.toUpperCase(); + return out_str; +} + +module.exports = { + randomItem, + generate_identifier, + generate_random_code, +}; + diff --git a/src/backend/src/util/linux.js b/src/backend/src/modules/core/lib/linux.js similarity index 84% rename from src/backend/src/util/linux.js rename to src/backend/src/modules/core/lib/linux.js index 546be5617b..5695a5f1d1 100644 --- a/src/backend/src/util/linux.js +++ b/src/backend/src/modules/core/lib/linux.js @@ -16,7 +16,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -const SmolUtil = require("./smolutil"); +const smol = require('@heyputer/putility').libs.smol; const parse_meminfo = text => { const lines = text.split('\n'); @@ -26,8 +26,8 @@ const parse_meminfo = text => { for ( const line of lines ) { if ( line.trim().length == 0 ) continue; - const [key, value_and_unit] = SmolUtil.split(line, ':', { trim: true }); - const [value, _] = SmolUtil.split(value_and_unit, ' ', { trim: true }); + const [key, value_and_unit] = smol.split(line, ':', { trim: true }); + const [value, _] = smol.split(value_and_unit, ' ', { trim: true }); // note: unit is always 'kB' so we discard it meminfo[key] = Number.parseInt(value); } @@ -38,3 +38,4 @@ const parse_meminfo = text => { module.exports = { parse_meminfo, }; + diff --git a/src/backend/src/modules/core/lib/log.js b/src/backend/src/modules/core/lib/log.js new file mode 100644 index 0000000000..f82a4b9d40 --- /dev/null +++ b/src/backend/src/modules/core/lib/log.js @@ -0,0 +1,73 @@ +// METADATA // {"def":"core.util.logutil","ai-commented":{"service":"openai-completion","model":"gpt-4o"}} +/* + * Copyright (C) 2024 Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +const log_epoch = Date.now(); + +/** +* Stringifies a log entry into a formatted string for console output. +* @param {Object} logEntry - The log entry object containing: +* @param {string} [prefix] - Optional prefix for the log message. +* @param {Object} log_lvl - Log level object with properties for label, escape code, etc. +* @param {string[]} crumbs - Array of context crumbs. +* @param {string} message - The log message. +* @param {Object} fields - Additional fields to be included in the log. +* @param {Object} objects - Objects to be logged. +* @returns {string} A formatted string representation of the log entry. +*/ +const stringify_log_entry = ({ prefix, log_lvl, crumbs, message, fields, objects }) => { + const { colorize } = require('json-colorizer'); + + let lines = [], m; + + const lf = () => { + if ( ! m ) return; + lines.push(m); + m = ''; + } + + m = prefix ? `${prefix} ` : ''; + m += `\x1B[${log_lvl.esc}m[${log_lvl.label}\x1B[0m`; + for ( const crumb of crumbs ) { + m += `::${crumb}`; + } + m += `\x1B[${log_lvl.esc}m]\x1B[0m`; + if ( fields.timestamp ) { + // display seconds since logger epoch + const n = (fields.timestamp - log_epoch) / 1000; + m += ` (${n.toFixed(3)}s)`; + } + m += ` ${message} `; + lf(); + for ( const k in fields ) { + if ( k === 'timestamp' ) continue; + let v; try { + v = colorize(JSON.stringify(fields[k])); + } catch (e) { + v = '' + fields[k]; + } + m += ` \x1B[1m${k}:\x1B[0m ${v}`; + lf(); + } + return lines.join('\n'); +}; + +module.exports = { + stringify_log_entry, + log_epoch, +}; diff --git a/src/backend/src/modules/core/lib/stdio.js b/src/backend/src/modules/core/lib/stdio.js new file mode 100644 index 0000000000..e2eadbcde7 --- /dev/null +++ b/src/backend/src/modules/core/lib/stdio.js @@ -0,0 +1,70 @@ +// METADATA // {"ai-commented":{"service":"claude"}} +/* + * Copyright (C) 2024 Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +/** + * Strip ANSI escape sequences from a string (e.g. color codes) + * and then return the length of the resulting string. + * + * @param {string} str - The string to calculate visible length for + * @returns {number} The length of the string without ANSI escape sequences + */ +const visible_length = (str) => { + // eslint-disable-next-line no-control-regex + return str.replace(/\x1b\[[0-9;]*m/g, '').length; +}; + +/** + * Split a string into lines according to the terminal width, + * preserving ANSI escape sequences, and return an array of lines. + * + * @param {string} str The string to split into lines + * @returns {string[]} Array of lines split according to terminal width + */ +const split_lines = (str) => { + const lines = []; + let line = ''; + let line_length = 0; + for (const c of str) { + line += c; + if (c === '\n') { + lines.push(line); + line = ''; + line_length = 0; + } else { + line_length++; + if (line_length >= process.stdout.columns) { + lines.push(line); + line = ''; + line_length = 0; + } + } + } + if (line.length) { + lines.push(line); + } + return lines; +}; + + +module.exports = { + visible_length, + split_lines, +}; + diff --git a/src/backend/src/modules/perfmon/PerfMonModule.js b/src/backend/src/modules/perfmon/PerfMonModule.js new file mode 100644 index 0000000000..02e531fbc9 --- /dev/null +++ b/src/backend/src/modules/perfmon/PerfMonModule.js @@ -0,0 +1,20 @@ +const { AdvancedBase } = require("@heyputer/putility"); + +/** + * Enable this module when you want performance monitoring. + * + * Performance monitoring requires additional setup. Jaegar should be installed + * and running. + */ +class PerfMonModule extends AdvancedBase { + async install (context) { + const services = context.get('services'); + + const TelemetryService = require("./TelemetryService"); + services.registerService('telemetry', TelemetryService); + } +} + +module.exports = { + PerfMonModule, +}; diff --git a/src/backend/src/monitor/Telemetry.js b/src/backend/src/modules/perfmon/TelemetryService.js similarity index 67% rename from src/backend/src/monitor/Telemetry.js rename to src/backend/src/modules/perfmon/TelemetryService.js index 60aaf7a99c..6ba040ad7f 100644 --- a/src/backend/src/monitor/Telemetry.js +++ b/src/backend/src/modules/perfmon/TelemetryService.js @@ -16,6 +16,7 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ +const opentelemetry = require("@opentelemetry/api"); const { NodeSDK } = require('@opentelemetry/sdk-node'); const { getNodeAutoInstrumentations } = require('@opentelemetry/auto-instrumentations-node'); const { PeriodicExportingMetricReader, ConsoleMetricExporter } = require('@opentelemetry/sdk-metrics'); @@ -25,17 +26,13 @@ const { SemanticResourceAttributes } = require("@opentelemetry/semantic-conventi const { NodeTracerProvider } = require("@opentelemetry/sdk-trace-node"); const { ConsoleSpanExporter, BatchSpanProcessor } = require("@opentelemetry/sdk-trace-base"); const { JaegerExporter } = require('@opentelemetry/exporter-jaeger'); -const config = require('../config'); +const config = require('../../config'); const { OTLPTraceExporter } = require('@opentelemetry/exporter-trace-otlp-grpc'); -class TelemetryService { - static instance_ = null; - static getInstance () { - if ( this.instance_ ) return this.instance_; - return this.instance_ = new TelemetryService(); - } +const BaseService = require('../../services/BaseService'); - constructor () { +class TelemetryService extends BaseService { + _construct () { const resource = Resource.default().merge( new Resource({ [SemanticResourceAttributes.SERVICE_NAME]: "puter-backend", @@ -61,6 +58,33 @@ class TelemetryService { }); this.sdk = sdk; + + this.sdk.start(); + + this.tracer_ = opentelemetry.trace.getTracer( + 'puter-tracer' + ); + } + + _init () { + const svc_context = this.services.get('context'); + svc_context.register_context_hook('pre_arun', ({ hints, trace_name, callback, replace_callback }) => { + if ( ! trace_name ) return; + if ( ! hints.trace ) return; + console.log('APPLYING TRACE NAME', trace_name); + replace_callback(async () => { + return await this.tracer_.startActiveSpan(trace_name, async span => { + try { + return await callback(); + } catch (error) { + span.setStatus({ code: opentelemetry.SpanStatusCode.ERROR, message: error.message }); + throw error; + } finally { + span.end(); + } + }); + }); + }); } getConfiguredExporter_() { @@ -69,12 +93,6 @@ class TelemetryService { } const exporter = new ConsoleSpanExporter(); } - - start () { - // this.sdk.start(); - } } -module.exports = { - TelemetryService -} \ No newline at end of file +module.exports = TelemetryService; diff --git a/src/backend/src/modules/puterai/AIChatService.js b/src/backend/src/modules/puterai/AIChatService.js index 317a6ef298..9bfd6259c7 100644 --- a/src/backend/src/modules/puterai/AIChatService.js +++ b/src/backend/src/modules/puterai/AIChatService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const APIError = require("../../api/APIError"); const { PermissionUtil } = require("../../services/auth/PermissionService"); const BaseService = require("../../services/BaseService"); @@ -5,15 +6,33 @@ const { DB_WRITE } = require("../../services/database/consts"); const { TypeSpec } = require("../../services/drivers/meta/Construct"); const { TypedValue } = require("../../services/drivers/meta/Runtime"); const { Context } = require("../../util/context"); +const { AsModeration } = require("./lib/AsModeration"); +// Maximum number of fallback attempts when a model fails, including the first attempt const MAX_FALLBACKS = 3 + 1; // includes first attempt + +/** +* AIChatService class extends BaseService to provide AI chat completion functionality. +* Manages multiple AI providers, models, and fallback mechanisms for chat interactions. +* Handles model registration, usage tracking, cost calculation, content moderation, +* and implements the puter-chat-completion driver interface. Supports streaming responses +* and maintains detailed model information including pricing and capabilities. +*/ class AIChatService extends BaseService { static MODULES = { kv: globalThis.kv, uuidv4: require('uuid').v4, } + + /** + * Initializes the service by setting up core properties. + * Creates empty arrays for providers and model lists, + * and initializes an empty object for the model map. + * Called during service instantiation. + * @private + */ _construct () { this.providers = []; @@ -21,6 +40,13 @@ class AIChatService extends BaseService { this.detail_model_list = []; this.detail_model_map = {}; } + /** + * Initializes the service by setting up empty arrays and maps for providers and models. + * This method is called during service construction to establish the initial state. + * Creates empty arrays for providers, simple model list, and detailed model list, + * as well as an empty object for the detailed model map. + * @private + */ _init () { this.kvkey = this.modules.uuidv4(); @@ -28,6 +54,8 @@ class AIChatService extends BaseService { const svc_event = this.services.get('event'); svc_event.on('ai.prompt.report-usage', async (_, details) => { + if ( details.service_used === 'fake-chat' ) return; + const values = { user_id: details.actor?.type?.user?.id, app_id: details.actor?.type?.app?.id ?? null, @@ -65,8 +93,32 @@ class AIChatService extends BaseService { await this.db.insert('ai_usage', values); }); + + const svc_apiErrpr = this.services.get('api-error'); + svc_apiErrpr.register({ + max_tokens_exceeded: { + status: 400, + message: ({ input_tokens, max_tokens }) => + `Input exceeds maximum token count. ` + + `Input has ${input_tokens} tokens, ` + + `but the maximum is ${max_tokens}.`, + }, + }); } + + /** + * Handles consolidation during service boot by registering service aliases + * and populating model lists/maps from providers. + * + * Registers each provider as an 'ai-chat' service alias and fetches their + * available models and pricing information. Populates: + * - simple_model_list: Basic list of supported models + * - detail_model_list: Detailed model info including costs + * - detail_model_map: Maps model IDs/aliases to their details + * + * @returns {Promise} + */ async ['__on_boot.consolidation'] () { { const svc_driver = this.services.get('driver') @@ -76,13 +128,21 @@ class AIChatService extends BaseService { } } - // TODO: get models and pricing for each model for ( const provider of this.providers ) { const delegate = this.services.get(provider.service_name) .as('puter-chat-completion'); // Populate simple model list { + /** + * Populates the simple model list by fetching available models from the delegate service. + * Wraps the delegate.list() call in a try-catch block to handle potential errors gracefully. + * If the call fails, logs the error and returns an empty array to avoid breaking the service. + * The fetched models are added to this.simple_model_list. + * + * @private + * @returns {Promise} + */ const models = await (async () => { try { return await delegate.list() ?? []; @@ -96,6 +156,14 @@ class AIChatService extends BaseService { // Populate detail model list and map { + /** + * Populates the detail model list and map with model information from the provider. + * Fetches detailed model data including pricing and capabilities. + * Handles model aliases and potential conflicts by storing multiple models in arrays. + * Annotates models with their provider service name. + * Catches and logs any errors during model fetching. + * @private + */ const models = await (async () => { try { return await delegate.models() ?? []; @@ -112,6 +180,13 @@ class AIChatService extends BaseService { }); } this.detail_model_list.push(...annotated_models); + /** + * Helper function to set or push a model into the detail_model_map. + * If there's no existing entry for the key, sets it directly. + * If there's a conflict, converts the entry to an array and pushes the new model. + * @param {string} key - The model ID or alias + * @param {Object} model - The model details to add + */ const set_or_push = (key, model) => { // Typical case: no conflict if ( ! this.detail_model_map[key] ) { @@ -153,16 +228,46 @@ class AIChatService extends BaseService { } }, ['puter-chat-completion']: { + /** + * Implements the 'puter-chat-completion' interface methods for AI chat functionality. + * Handles model selection, fallbacks, usage tracking, and moderation. + * Contains methods for listing available models, completing chat prompts, + * and managing provider interactions. + * + * @property {Object} models - Available AI models with details like costs + * @property {Object} list - Simplified list of available models + * @property {Object} complete - Main method for chat completion requests + * @param {Object} parameters - Chat completion parameters including model and messages + * @returns {Promise} Chat completion response with usage stats + * @throws {Error} If service is called directly or no fallback models available + */ async models () { const delegate = this.get_delegate(); if ( ! delegate ) return await this.models_(); return await delegate.models(); }, + /** + * Returns list of available AI models with detailed information + * + * Delegates to the intended service's models() method if a delegate exists, + * otherwise returns the internal detail_model_list containing all available models + * across providers with their capabilities and pricing information. + * + * @returns {Promise} Array of model objects with details like id, provider, cost, etc. + */ async list () { const delegate = this.get_delegate(); if ( ! delegate ) return await this.list_(); return await delegate.list(); }, + /** + * Lists available AI models in a simplified format + * + * Returns a list of basic model information from all registered providers. + * This is a simpler version compared to models() that returns less detailed info. + * + * @returns {Promise} Array of simplified model objects + */ async complete (parameters) { const client_driver_call = Context.get('client_driver_call'); let { test_mode, intended_service, response_metadata } = client_driver_call; @@ -174,6 +279,7 @@ class AIChatService extends BaseService { intended_service, parameters }; + await svc_event.emit('ai.prompt.validate', event); if ( ! event.allow ) { test_mode = true; } @@ -188,6 +294,9 @@ class AIChatService extends BaseService { if ( test_mode ) { intended_service = 'fake-chat'; + if ( event.abuse ) { + parameters.model = 'abuse'; + } } if ( intended_service === this.service_name ) { @@ -195,9 +304,11 @@ class AIChatService extends BaseService { } const svc_driver = this.services.get('driver'); - let ret, error, errors = []; + let ret, error; let service_used = intended_service; - let model_used = this.get_model_from_request(parameters); + let model_used = this.get_model_from_request(parameters, { + intended_service + }); await this.check_usage_({ actor: Context.get('actor'), service: service_used, @@ -219,7 +330,6 @@ class AIChatService extends BaseService { tried.push(model); error = e; - errors.push(e); console.error(e); this.log.error('error calling service', { intended_service, @@ -272,7 +382,6 @@ class AIChatService extends BaseService { }; } catch (e) { error = e; - errors.push(e); tried.push(fallback_model_name); this.log.error('error calling fallback', { intended_service, @@ -330,6 +439,17 @@ class AIChatService extends BaseService { } } + + /** + * Checks if the user has permission to use AI services and verifies usage limits + * + * @param {Object} params - The check parameters + * @param {Object} params.actor - The user/actor making the request + * @param {string} params.service - The AI service being used + * @param {string} params.model - The model being accessed + * @throws {APIError} If usage is not allowed or limits are exceeded + * @private + */ async check_usage_ ({ actor, service, model }) { const svc_permission = this.services.get('permission'); const svc_event = this.services.get('event'); @@ -359,12 +479,21 @@ class AIChatService extends BaseService { } } - async moderate ({ messages }) { - const svc_openai = this.services.get('openai-completion'); - // We can't use moderation of openai service isn't available - if ( ! svc_openai ) return true; - + /** + * Moderates chat messages for inappropriate content using OpenAI's moderation service + * + * @param {Object} params - The parameters object + * @param {Array} params.messages - Array of chat messages to moderate + * @returns {Promise} Returns true if content is appropriate, false if flagged + * + * @description + * Extracts text content from messages and checks each against OpenAI's moderation. + * Handles both string content and structured message objects. + * Returns false immediately if any message is flagged as inappropriate. + * Returns true if OpenAI service is unavailable or all messages pass moderation. + */ + async moderate ({ messages }) { for ( const msg of messages ) { const texts = []; if ( typeof msg.content === 'string' ) texts.push(msg.content); @@ -379,20 +508,67 @@ class AIChatService extends BaseService { const fulltext = texts.join('\n'); - const mod_result = await svc_openai.check_moderation(fulltext); - if ( mod_result.flagged ) return false; + let mod_last_error = null; + let mod_result = null; + try { + const svc_openai = this.services.get('openai-completion'); + mod_result = await svc_openai.check_moderation(fulltext); + if ( mod_result.flagged ) return false; + continue; + } catch (e) { + console.error(e); + mod_last_error = e; + } + try { + const svc_claude = this.services.get('claude'); + const chat = svc_claude.as('puter-chat-completion'); + const mod = new AsModeration({ + chat, + model: 'claude-3-haiku-20240307', + }) + if ( ! await mod.moderate(fulltext) ) { + return false; + } + mod_last_error = null; + continue; + } catch (e) { + console.error(e); + mod_last_error = e; + } + + if ( mod_last_error ) { + this.log.error('moderation error', { + fulltext, + mod_last_error, + }); + throw new Error('no working moderation service'); + } } return true; } + async models_ () { return this.detail_model_list; } + + /** + * Returns a list of available AI models with basic details + * @returns {Promise} Array of simple model objects containing basic model information + */ async list_ () { return this.simple_model_list; } + + /** + * Gets the appropriate delegate service for handling chat completion requests. + * If the intended service is this service (ai-chat), returns undefined. + * Otherwise returns the intended service wrapped as a puter-chat-completion interface. + * + * @returns {Object|undefined} The delegate service or undefined if intended service is ai-chat + */ get_delegate () { const client_driver_call = Context.get('client_driver_call'); if ( client_driver_call.intended_service === this.service_name ) { @@ -431,7 +607,7 @@ class AIChatService extends BaseService { // Calculate the sorted list const models = this.detail_model_list; - sorted_models = models.sort((a, b) => { + sorted_models = models.toSorted((a, b) => { return Math.sqrt( Math.pow(a.cost.input - target_model.cost.input, 2) + Math.pow(a.cost.output - target_model.cost.output, 2) @@ -463,9 +639,13 @@ class AIChatService extends BaseService { }); } - get_model_from_request (parameters) { + get_model_from_request (parameters, modified_context = {}) { const client_driver_call = Context.get('client_driver_call'); let { intended_service } = client_driver_call; + + if ( modified_context.intended_service ) { + intended_service = modified_context.intended_service; + } let model = parameters.model; if ( ! model ) { diff --git a/src/backend/src/modules/puterai/AIInterfaceService.js b/src/backend/src/modules/puterai/AIInterfaceService.js index 17052cbdae..33bdffc24f 100644 --- a/src/backend/src/modules/puterai/AIInterfaceService.js +++ b/src/backend/src/modules/puterai/AIInterfaceService.js @@ -1,6 +1,20 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const BaseService = require("../../services/BaseService"); + +/** +* Service class that manages AI interface registrations and configurations. +* Handles registration of various AI services including OCR, chat completion, +* image generation, and text-to-speech interfaces. Each interface defines +* its available methods, parameters, and expected results. +* @extends BaseService +*/ class AIInterfaceService extends BaseService { + /** + * Service class for managing AI interface registrations and configurations. + * Extends the base service to provide AI-related interface management. + * Handles registration of OCR, chat completion, image generation, and TTS interfaces. + */ async ['__on_driver.register.interfaces'] () { const svc_registry = this.services.get('registry'); const col_interfaces = svc_registry.get('interfaces'); diff --git a/src/backend/src/modules/puterai/AITestModeService.js b/src/backend/src/modules/puterai/AITestModeService.js index 97dbfc385e..c01a78a735 100644 --- a/src/backend/src/modules/puterai/AITestModeService.js +++ b/src/backend/src/modules/puterai/AITestModeService.js @@ -1,6 +1,18 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const BaseService = require("../../services/BaseService"); + +/** +* Service class that handles AI test mode functionality. +* Extends BaseService to register test services for AI chat completions. +* Used for testing and development of AI-related features by providing +* a mock implementation of the chat completion service. +*/ class AITestModeService extends BaseService { + /** + * Service for managing AI test mode functionality + * @extends BaseService + */ async _init () { const svc_driver = this.services.get('driver'); svc_driver.register_test_service('puter-chat-completion', 'ai-chat'); diff --git a/src/backend/src/modules/puterai/AWSPollyService.js b/src/backend/src/modules/puterai/AWSPollyService.js index 5189e2592f..2fd72f26e2 100644 --- a/src/backend/src/modules/puterai/AWSPollyService.js +++ b/src/backend/src/modules/puterai/AWSPollyService.js @@ -1,12 +1,28 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const { PollyClient, SynthesizeSpeechCommand, DescribeVoicesCommand } = require("@aws-sdk/client-polly"); const BaseService = require("../../services/BaseService"); const { TypedValue } = require("../../services/drivers/meta/Runtime"); + +/** +* AWSPollyService class provides text-to-speech functionality using Amazon Polly. +* Extends BaseService to integrate with AWS Polly for voice synthesis operations. +* Implements voice listing, speech synthesis, and voice selection based on language. +* Includes caching for voice descriptions and supports both text and SSML inputs. +* @extends BaseService +*/ class AWSPollyService extends BaseService { static MODULES = { kv: globalThis.kv, } + + /** + * Initializes the service by creating an empty clients object. + * This method is called during service construction to set up + * the internal state needed for AWS Polly client management. + * @returns {Promise} + */ async _construct () { this.clients_ = {}; } @@ -18,6 +34,14 @@ class AWSPollyService extends BaseService { } }, ['puter-tts']: { + /** + * Implements the driver interface methods for text-to-speech functionality + * Contains methods for listing available voices and synthesizing speech + * @interface + * @property {Object} list_voices - Lists available Polly voices with language info + * @property {Object} synthesize - Converts text to speech using specified voice/language + * @property {Function} supports_test_mode - Indicates test mode support for methods + */ async list_voices () { const polly_voices = await this.describe_voices(); @@ -64,6 +88,12 @@ class AWSPollyService extends BaseService { } } + + /** + * Creates AWS credentials object for authentication + * @private + * @returns {Object} Object containing AWS access key ID and secret access key + */ _create_aws_credentials () { return { accessKeyId: this.config.aws.access_key, @@ -86,6 +116,13 @@ class AWSPollyService extends BaseService { return this.clients_[region]; } + + /** + * Describes available AWS Polly voices and caches the results + * @returns {Promise} Response containing array of voice details in Voices property + * @description Fetches voice information from AWS Polly API and caches it for 10 minutes + * Uses KV store for caching to avoid repeated API calls + */ async describe_voices () { let voices = this.modules.kv.get('svc:polly:voices'); if ( voices ) { @@ -109,6 +146,17 @@ class AWSPollyService extends BaseService { return response; } + + /** + * Synthesizes speech from text using AWS Polly + * @param {string} text - The text to synthesize + * @param {Object} options - Synthesis options + * @param {string} options.format - Output audio format (e.g. 'mp3') + * @param {string} [options.voice_id] - AWS Polly voice ID to use + * @param {string} [options.language] - Language code (e.g. 'en-US') + * @param {string} [options.text_type] - Type of input text ('text' or 'ssml') + * @returns {Promise} The synthesized speech response + */ async synthesize_speech (text, { format, voice_id, language, text_type }) { const client = this._get_client(this.config.aws.region); @@ -140,6 +188,13 @@ class AWSPollyService extends BaseService { return response; } + + /** + * Attempts to find an appropriate voice for the given language code + * @param {string} language - The language code to find a voice for (e.g. 'en-US') + * @returns {Promise} The voice ID if found, null if no matching voice exists + * @private + */ async maybe_get_language_appropriate_voice_ (language) { const voices = await this.describe_voices(); diff --git a/src/backend/src/modules/puterai/AWSTextractService.js b/src/backend/src/modules/puterai/AWSTextractService.js index e216397a56..4b277694c7 100644 --- a/src/backend/src/modules/puterai/AWSTextractService.js +++ b/src/backend/src/modules/puterai/AWSTextractService.js @@ -1,9 +1,23 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const { TextractClient, AnalyzeDocumentCommand, InvalidS3ObjectException } = require("@aws-sdk/client-textract"); const BaseService = require("../../services/BaseService"); const APIError = require("../../api/APIError"); + +/** +* AWSTextractService class - Provides OCR (Optical Character Recognition) functionality using AWS Textract +* Extends BaseService to integrate with AWS Textract for document analysis and text extraction. +* Implements driver capabilities and puter-ocr interface for document recognition. +* Handles both S3-stored and buffer-based document processing with automatic region management. +*/ class AWSTextractService extends BaseService { + /** + * AWS Textract service for OCR functionality + * Provides document analysis capabilities using AWS Textract API + * Implements interfaces for OCR recognition and driver capabilities + * @extends BaseService + */ _construct () { this.clients_ = {}; } @@ -15,6 +29,13 @@ class AWSTextractService extends BaseService { } }, ['puter-ocr']: { + /** + * Performs OCR recognition on a document using AWS Textract + * @param {Object} params - Recognition parameters + * @param {Object} params.source - The document source to analyze + * @param {boolean} params.test_mode - If true, returns sample test output instead of processing + * @returns {Promise} Recognition results containing blocks of text with confidence scores + */ async recognize ({ source, test_mode }) { if ( test_mode ) { return { @@ -61,6 +82,12 @@ class AWSTextractService extends BaseService { }, }; + + /** + * Creates AWS credentials object for authentication + * @private + * @returns {Object} Object containing AWS access key ID and secret access key + */ _create_aws_credentials () { return { accessKeyId: this.config.aws.access_key, @@ -83,6 +110,15 @@ class AWSTextractService extends BaseService { return this.clients_[region]; } + + /** + * Analyzes a document using AWS Textract to extract text and layout information + * @param {FileFacade} file_facade - Interface to access the document file + * @returns {Promise} The raw Textract API response containing extracted text blocks + * @throws {Error} If document analysis fails or no suitable input format is available + * @description Processes document through Textract's AnalyzeDocument API with LAYOUT feature. + * Will attempt to use S3 direct access first, falling back to buffer upload if needed. + */ async analyze_document (file_facade) { const { client, document, using_s3 @@ -119,6 +155,18 @@ class AWSTextractService extends BaseService { throw new Error('expected to be unreachable'); } + + /** + * Gets AWS client and document configuration for Textract processing + * @param {Object} file_facade - File facade object containing document source info + * @param {boolean} [force_buffer] - If true, forces using buffer instead of S3 + * @returns {Promise} Object containing: + * - client: Configured AWS Textract client + * - document: Document configuration for Textract + * - using_s3: Boolean indicating if using S3 source + * @throws {APIError} If file does not exist + * @throws {Error} If no suitable input format is available + */ async _get_client_and_document (file_facade, force_buffer) { const try_s3info = await file_facade.get('s3-info'); if ( try_s3info && ! force_buffer ) { @@ -137,7 +185,6 @@ class AWSTextractService extends BaseService { const try_buffer = await file_facade.get('buffer'); if ( try_buffer ) { - const base64 = try_buffer.toString('base64'); return { client: this._get_client(), document: { diff --git a/src/backend/src/modules/puterai/ClaudeEnoughService.js b/src/backend/src/modules/puterai/ClaudeEnoughService.js index e1b57bba83..c844f77be6 100644 --- a/src/backend/src/modules/puterai/ClaudeEnoughService.js +++ b/src/backend/src/modules/puterai/ClaudeEnoughService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const { XAIService } = require("./XAIService"); const CLAUDE_ENOUGH_PROMPT = ` @@ -19,7 +20,20 @@ const CLAUDE_ENOUGH_PROMPT = ` user of the driver interface (typically an app on Puter): `.replace('\n', ' ').trim(); + +/** +* ClaudeEnoughService - A service class that implements a Claude-like AI interface +* Extends XAIService to provide Claude-compatible responses while using alternative AI models. +* Includes custom system prompts and model adaptation to simulate Claude's behavior +* in the Puter platform's chat completion interface. +*/ class ClaudeEnoughService extends XAIService { + /** + * Service that emulates Claude's behavior using alternative AI models + * @extends XAIService + * @description Provides a Claude-like interface while using other AI models as the backend. + * Includes custom system prompts and model adaptations to approximate Claude's behavior. + */ get_system_prompt () { return CLAUDE_ENOUGH_PROMPT; } diff --git a/src/backend/src/modules/puterai/ClaudeService.js b/src/backend/src/modules/puterai/ClaudeService.js index 98b14c18b0..a47c517785 100644 --- a/src/backend/src/modules/puterai/ClaudeService.js +++ b/src/backend/src/modules/puterai/ClaudeService.js @@ -1,10 +1,11 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const { default: Anthropic } = require("@anthropic-ai/sdk"); const BaseService = require("../../services/BaseService"); const { whatis } = require("../../util/langutil"); const { PassThrough } = require("stream"); const { TypedValue } = require("../../services/drivers/meta/Runtime"); const APIError = require("../../api/APIError"); -const { TeePromise } = require("../../util/promise"); +const { TeePromise } = require('@heyputer/putility').libs.promise; const PUTER_PROMPT = ` You are running on an open-source platform called Puter, @@ -15,13 +16,27 @@ const PUTER_PROMPT = ` user of the driver interface (typically an app on Puter): `.replace('\n', ' ').trim(); -const MAX_CLAUDE_INPUT_TOKENS = 10000; + +/** +* ClaudeService class extends BaseService to provide integration with Anthropic's Claude AI models. +* Implements the puter-chat-completion interface for handling AI chat interactions. +* Manages message streaming, token limits, model selection, and API communication with Claude. +* Supports system prompts, message adaptation, and usage tracking. +* @extends BaseService +*/ class ClaudeService extends BaseService { static MODULES = { Anthropic: require('@anthropic-ai/sdk'), } + + /** + * Initializes the Claude service by creating an Anthropic client instance + * and registering this service as a provider with the AI chat service. + * @private + * @returns {Promise} + */ async _init () { this.anthropic = new Anthropic({ apiKey: this.config.apiKey @@ -34,15 +49,34 @@ class ClaudeService extends BaseService { }); } + + /** + * Returns the default model identifier for Claude API interactions + * @returns {string} The default model ID 'claude-3-5-sonnet-latest' + */ get_default_model () { return 'claude-3-5-sonnet-latest'; } static IMPLEMENTS = { ['puter-chat-completion']: { + /** + * Implements the puter-chat-completion interface for Claude AI models + * @param {Object} options - Configuration options for the chat completion + * @param {Array} options.messages - Array of message objects containing the conversation history + * @param {boolean} options.stream - Whether to stream the response + * @param {string} [options.model] - The Claude model to use, defaults to claude-3-5-sonnet-latest + * @returns {TypedValue|Object} Returns either a TypedValue with streaming response or a completion object + */ async models () { return await this.models_(); }, + /** + * Returns a list of available model names including their aliases + * @returns {Promise} Array of model identifiers and their aliases + * @description Retrieves all available Claude model IDs and their aliases, + * flattening them into a single array of strings that can be used for model selection + */ async list () { const models = await this.models_(); const model_names = []; @@ -54,6 +88,14 @@ class ClaudeService extends BaseService { } return model_names; }, + /** + * Completes a chat interaction with the Claude AI model + * @param {Object} options - The completion options + * @param {Array} options.messages - Array of chat messages to process + * @param {boolean} options.stream - Whether to stream the response + * @param {string} [options.model] - The Claude model to use, defaults to service default + * @returns {TypedValue|Object} Returns either a TypedValue with streaming response or a completion object + */ async complete ({ messages, stream, model }) { const adapted_messages = []; @@ -84,24 +126,11 @@ class ClaudeService extends BaseService { adapted_messages.push(message); if ( message.role === 'user' ) { previous_was_user = true; + } else { + previous_was_user = false; } } - const token_count = (() => { - const text = JSON.stringify(adapted_messages) + - JSON.stringify(system_prompts); - - // This is the most accurate token counter available for Claude. - return text.length / 4; - })(); - - if ( token_count > MAX_CLAUDE_INPUT_TOKENS ) { - throw APIError.create('max_tokens_exceeded', null, { - input_tokens: token_count, - max_tokens: MAX_CLAUDE_INPUT_TOKENS, - }); - } - if ( stream ) { let usage_promise = new TeePromise(); @@ -114,7 +143,7 @@ class ClaudeService extends BaseService { (async () => { const completion = await this.anthropic.messages.stream({ model: model ?? this.get_default_model(), - max_tokens: 1000, + max_tokens: (model === 'claude-3-5-sonnet-20241022' || model === 'claude-3-5-sonnet-20240620') ? 8192 : 4096, temperature: 0, system: PUTER_PROMPT + JSON.stringify(system_prompts), messages: adapted_messages, @@ -151,7 +180,7 @@ class ClaudeService extends BaseService { const msg = await this.anthropic.messages.create({ model: model ?? this.get_default_model(), - max_tokens: 1000, + max_tokens: (model === 'claude-3-5-sonnet-20241022' || model === 'claude-3-5-sonnet-20240620') ? 8192 : 4096, temperature: 0, system: PUTER_PROMPT + JSON.stringify(system_prompts), messages: adapted_messages, @@ -165,6 +194,19 @@ class ClaudeService extends BaseService { } } + + /** + * Retrieves available Claude AI models and their specifications + * @returns {Promise} Array of model objects containing: + * - id: Model identifier + * - name: Display name + * - aliases: Alternative names for the model + * - context: Maximum context window size + * - cost: Pricing details (currency, token counts, input/output costs) + * - qualitative_speed: Relative speed rating + * - max_output: Maximum output tokens + * - training_cutoff: Training data cutoff date + */ async models_ () { return [ { diff --git a/src/backend/src/modules/puterai/FakeChatService.js b/src/backend/src/modules/puterai/FakeChatService.js index 500f9d7d5b..ff157fdd4d 100644 --- a/src/backend/src/modules/puterai/FakeChatService.js +++ b/src/backend/src/modules/puterai/FakeChatService.js @@ -1,11 +1,37 @@ +// METADATA // {"ai-commented":{"service":"claude"}} +const { default: dedent } = require("dedent"); const BaseService = require("../../services/BaseService"); + +/** +* FakeChatService - A mock implementation of a chat service that extends BaseService. +* Provides fake chat completion responses using Lorem Ipsum text generation. +* Used for testing and development purposes when a real chat service is not needed. +* Implements the 'puter-chat-completion' interface with list() and complete() methods. +*/ class FakeChatService extends BaseService { + get_default_model () { + return 'fake'; + } static IMPLEMENTS = { ['puter-chat-completion']: { + /** + * Implementation interface for the puter-chat-completion service. + * Provides fake chat completion functionality for testing purposes. + * Contains methods for listing available models and generating mock responses. + * @interface + */ async list () { return ['fake']; }, + /** + * Simulates a chat completion request by generating random Lorem Ipsum text + * @param {Object} params - The completion parameters + * @param {Array} params.messages - Array of chat messages (unused in fake implementation) + * @param {boolean} params.stream - Whether to stream the response (unused in fake implementation) + * @param {string} params.model - The model to use (unused in fake implementation) + * @returns {Object} A simulated chat completion response with Lorem Ipsum content + */ async complete ({ messages, stream, model }) { const { LoremIpsum } = require('lorem-ipsum'); const li = new LoremIpsum({ @@ -28,7 +54,13 @@ class FakeChatService extends BaseService { "content": [ { "type": "text", - "text": li.generateParagraphs( + "text": model === 'abuse' ? dedent(` + This is a message from ${ + this.global_config.origin}. We have detected abuse of our services. + + If you are seeing this on another website, please report it to ${ + this.global_config.abuse_email ?? 'hi@puter.com'} + `) : li.generateParagraphs( Math.floor(Math.random() * 3) + 1 ) } @@ -40,6 +72,10 @@ class FakeChatService extends BaseService { "output_tokens": 1 } }, + "usage": { + "input_tokens": 0, + "output_tokens": 1 + }, "logprobs": null, "finish_reason": "stop" } diff --git a/src/backend/src/modules/puterai/GroqAIService.js b/src/backend/src/modules/puterai/GroqAIService.js index daf7d0f510..454aad74af 100644 --- a/src/backend/src/modules/puterai/GroqAIService.js +++ b/src/backend/src/modules/puterai/GroqAIService.js @@ -1,14 +1,31 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const { PassThrough } = require("stream"); const BaseService = require("../../services/BaseService"); const { TypedValue } = require("../../services/drivers/meta/Runtime"); const { nou } = require("../../util/langutil"); -const { TeePromise } = require("../../util/promise"); +const { TeePromise } = require('@heyputer/putility').libs.promise; + +/** +* Service class for integrating with Groq AI's language models. +* Extends BaseService to provide chat completion capabilities through the Groq API. +* Implements the puter-chat-completion interface for model management and text generation. +* Supports both streaming and non-streaming responses, handles multiple models including +* various versions of Llama, Mixtral, and Gemma, and manages usage tracking. +* @class GroqAIService +* @extends BaseService +*/ class GroqAIService extends BaseService { static MODULES = { Groq: require('groq-sdk'), } + + /** + * Initializes the GroqAI service by setting up the Groq client and registering with the AI chat provider + * @returns {Promise} + * @private + */ async _init () { const Groq = require('groq-sdk'); this.client = new Groq({ @@ -22,20 +39,47 @@ class GroqAIService extends BaseService { }); } + + /** + * Returns the default model ID for the Groq AI service + * @returns {string} The default model ID 'llama-3.1-8b-instant' + */ get_default_model () { return 'llama-3.1-8b-instant'; } static IMPLEMENTS = { 'puter-chat-completion': { + /** + * Defines the interface implementations for the puter-chat-completion service + * Contains methods for listing models and handling chat completions + * @property {Object} models - Returns available AI models + * @property {Object} list - Lists raw model data from the Groq API + * @property {Object} complete - Handles chat completion requests with optional streaming + * @returns {Object} Interface implementation object + */ async models () { return await this.models_(); }, + /** + * Lists available AI models from the Groq API + * @returns {Promise} Array of model objects from the API's data field + * @description Unwraps and returns the model list from the Groq API response, + * which comes wrapped in an object with {object: "list", data: [...]} + */ async list () { // They send: { "object": "list", data } const funny_wrapper = await this.client.models.list(); return funny_wrapper.data; }, + /** + * Completes a chat interaction using the Groq API + * @param {Object} options - The completion options + * @param {Array} options.messages - Array of message objects containing the conversation history + * @param {string} [options.model] - The model ID to use for completion. Defaults to service's default model + * @param {boolean} [options.stream] - Whether to stream the response + * @returns {TypedValue|Object} Returns either a TypedValue with streaming response or completion object with usage stats + */ async complete ({ messages, model, stream }) { for ( let i = 0; i < messages.length; i++ ) { const message = messages[i]; @@ -101,6 +145,18 @@ class GroqAIService extends BaseService { } }; + + /** + * Returns an array of available AI models with their specifications + * + * Each model object contains: + * - id: Unique identifier for the model + * - name: Human-readable name + * - context: Maximum context window size in tokens + * - cost: Pricing details including currency and token rates + * + * @returns {Array} Array of model specification objects + */ models_ () { return [ { diff --git a/src/backend/src/modules/puterai/MistralAIService.js b/src/backend/src/modules/puterai/MistralAIService.js index 2f9caffb17..ccca80fbd8 100644 --- a/src/backend/src/modules/puterai/MistralAIService.js +++ b/src/backend/src/modules/puterai/MistralAIService.js @@ -1,15 +1,30 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const { PassThrough } = require("stream"); const BaseService = require("../../services/BaseService"); const { TypedValue } = require("../../services/drivers/meta/Runtime"); const { nou } = require("../../util/langutil"); const axios = require('axios'); -const { TeePromise } = require("../../util/promise"); +const { TeePromise } = require('@heyputer/putility').libs.promise; + +/** +* MistralAIService class extends BaseService to provide integration with the Mistral AI API. +* Implements chat completion functionality with support for various Mistral models including +* mistral-large, pixtral, codestral, and ministral variants. Handles both streaming and +* non-streaming responses, token usage tracking, and model management. Provides cost information +* for different models and implements the puter-chat-completion interface. +*/ class MistralAIService extends BaseService { static MODULES = { '@mistralai/mistralai': require('@mistralai/mistralai'), } + /** + * Initializes the service's cost structure for different Mistral AI models. + * Sets up pricing information for various models including token costs for input/output. + * Each model entry specifies currency (usd-cents) and costs per million tokens. + * @private + */ _construct () { this.costs_ = { 'mistral-large-latest': { @@ -80,6 +95,12 @@ class MistralAIService extends BaseService { }, }; } + /** + * Initializes the service's cost structure for different Mistral AI models. + * Sets up pricing information for various models including token costs for input/output. + * Each model entry specifies currency (USD cents) and costs per million tokens. + * @private + */ async _init () { const require = this.require; const { Mistral } = require('@mistralai/mistralai'); @@ -97,6 +118,13 @@ class MistralAIService extends BaseService { // TODO: make this event-driven so it doesn't hold up boot await this.populate_models_(); } + /** + * Populates the internal models array with available Mistral AI models and their configurations. + * Makes an API call to fetch model data, then processes and filters models based on cost information. + * Each model entry includes id, name, aliases, context window size, capabilities, and pricing. + * @private + * @returns {Promise} + */ async populate_models_ () { const resp = await axios({ method: 'get', @@ -131,17 +159,41 @@ class MistralAIService extends BaseService { } // return resp.data; } + /** + * Populates the internal models array with available Mistral AI models and their metadata + * Fetches model data from the API, filters based on cost configuration, and stores + * model objects containing ID, name, aliases, context length, capabilities, and pricing + * @private + * @async + * @returns {void} + */ get_default_model () { return 'mistral-large-latest'; } static IMPLEMENTS = { 'puter-chat-completion': { + /** + * Implements the puter-chat-completion interface for MistralAI service + * Provides methods for listing models and generating chat completions + * @interface + * @property {Function} models - Returns array of available model details + * @property {Function} list - Returns array of model IDs + * @property {Function} complete - Generates chat completion with optional streaming + */ async models () { return this.models_array_; }, + /** + * Returns an array of available AI models with their details + * @returns {Promise} Array of model objects containing id, name, aliases, context window size, capabilities, and cost information + */ async list () { return this.models_array_.map(m => m.id); }, + /** + * Returns an array of model IDs supported by the MistralAI service + * @returns {Promise} Array of model identifier strings + */ async complete ({ messages, stream, model }) { for ( let i = 0; i < messages.length; i++ ) { diff --git a/src/backend/src/modules/puterai/OpenAICompletionService.js b/src/backend/src/modules/puterai/OpenAICompletionService.js index 4735094b41..1bfed00e75 100644 --- a/src/backend/src/modules/puterai/OpenAICompletionService.js +++ b/src/backend/src/modules/puterai/OpenAICompletionService.js @@ -1,17 +1,33 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const { PassThrough } = require('stream'); const APIError = require('../../api/APIError'); const BaseService = require('../../services/BaseService'); const { TypedValue } = require('../../services/drivers/meta/Runtime'); const { Context } = require('../../util/context'); -const SmolUtil = require('../../util/smolutil'); +const smol = require('@heyputer/putility').libs.smol; const { nou } = require('../../util/langutil'); -const { TeePromise } = require('../../util/promise'); +const { TeePromise } = require('@heyputer/putility').libs.promise; + +/** +* OpenAICompletionService class provides an interface to OpenAI's chat completion API. +* Extends BaseService to handle chat completions, message moderation, token counting, +* and streaming responses. Implements the puter-chat-completion interface and manages +* OpenAI API interactions with support for multiple models including GPT-4 variants. +* Handles usage tracking, spending records, and content moderation. +*/ class OpenAICompletionService extends BaseService { static MODULES = { openai: require('openai'), tiktoken: require('tiktoken'), } + /** + * Initializes the OpenAI service by setting up the API client with credentials + * and registering this service as a chat provider. + * + * @returns {Promise} Resolves when initialization is complete + * @private + */ async _init () { const sk_key = this.config?.openai?.secret_key ?? @@ -28,10 +44,21 @@ class OpenAICompletionService extends BaseService { }); } + + /** + * Gets the default model identifier for OpenAI completions + * @returns {string} The default model ID 'gpt-4o-mini' + */ get_default_model () { return 'gpt-4o-mini'; } + + /** + * Returns an array of available AI models with their pricing information. + * Each model object includes an ID and cost details (currency, tokens, input/output rates). + * @returns {Promise} + */ async models_ () { return [ { @@ -75,9 +102,25 @@ class OpenAICompletionService extends BaseService { static IMPLEMENTS = { ['puter-chat-completion']: { + /** + * Implements the puter-chat-completion interface methods for model listing and chat completion + * @property {Object} models - Returns available AI models and their pricing + * @property {Function} list - Returns list of available model names/aliases + * @property {Function} complete - Handles chat completion requests with optional streaming + * @param {Object} params - Parameters for completion + * @param {Array} params.messages - Array of chat messages + * @param {boolean} params.test_mode - Whether to use test mode + * @param {boolean} params.stream - Whether to stream responses + * @param {string} params.model - Model ID to use + */ async models () { return await this.models_(); }, + /** + * Retrieves a list of available AI models with their cost information + * @returns {Promise} Array of model objects containing id and cost details + * @private + */ async list () { const models = await this.models_(); const model_names = []; @@ -89,6 +132,10 @@ class OpenAICompletionService extends BaseService { } return model_names; }, + /** + * Lists all available model names including aliases + * @returns {Promise} Array of model IDs and their aliases + */ async complete ({ messages, test_mode, stream, model }) { // for now this code (also in AIChatService.js) needs to be @@ -139,6 +186,14 @@ class OpenAICompletionService extends BaseService { } }; + + /** + * Checks text content against OpenAI's moderation API for inappropriate content + * @param {string} text - The text content to check for moderation + * @returns {Promise} Object containing flagged status and detailed results + * @property {boolean} flagged - Whether the content was flagged as inappropriate + * @property {Object} results - Raw moderation results from OpenAI API + */ async check_moderation (text) { // create moderation const results = await this.openai.moderations.create({ @@ -160,6 +215,17 @@ class OpenAICompletionService extends BaseService { }; } + + /** + * Completes a chat conversation using OpenAI's API + * @param {Array} messages - Array of message objects or strings representing the conversation + * @param {Object} options - Configuration options + * @param {boolean} options.stream - Whether to stream the response + * @param {boolean} options.moderation - Whether to perform content moderation + * @param {string} options.model - The model to use for completion + * @returns {Promise} The completion response containing message and usage info + * @throws {Error} If messages are invalid or content is flagged by moderation + */ async complete (messages, { stream, moderation, model }) { // Validate messages if ( ! Array.isArray(messages) ) { @@ -234,7 +300,7 @@ class OpenAICompletionService extends BaseService { if ( ! msg.content ) continue; if ( typeof msg.content !== 'object' ) continue; - const content = SmolUtil.ensure_array(msg.content); + const content = smol.ensure_array(msg.content); for ( const o of content ) { if ( ! o.hasOwnProperty('image_url') ) continue; @@ -260,7 +326,7 @@ class OpenAICompletionService extends BaseService { if ( ! msg.content ) continue; if ( typeof msg.content !== 'object' ) continue; - const content = SmolUtil.ensure_array(msg.content); + const content = smol.ensure_array(msg.content); for ( const o of content ) { // console.log('part of content', o); @@ -275,8 +341,9 @@ class OpenAICompletionService extends BaseService { const max_tokens = 4096 - token_count; console.log('MAX TOKENS ???', max_tokens); + const svc_apiErrpr = this.services.get('api-error'); if ( max_tokens <= 8 ) { - throw APIError.create('max_tokens_exceeded', null, { + throw svc_apiErrpr.create('max_tokens_exceeded', { input_tokens: token_count, max_tokens: 4096 - 8, }); @@ -338,6 +405,13 @@ class OpenAICompletionService extends BaseService { const spending_meta = {}; spending_meta.timestamp = Date.now(); spending_meta.count_tokens_input = token_count; + /** + * Records spending metadata for the chat completion request and performs token counting. + * Initializes metadata object with timestamp and token counts for both input and output. + * Uses tiktoken to count output tokens from the completion response. + * Records spending data via spending service and increments usage counters. + * @private + */ spending_meta.count_tokens_output = (() => { // count output tokens (overestimate) const enc = this.modules.tiktoken.encoding_for_model(model); diff --git a/src/backend/src/modules/puterai/OpenAIImageGenerationService.js b/src/backend/src/modules/puterai/OpenAIImageGenerationService.js index b4271e1246..8b3052cdaa 100644 --- a/src/backend/src/modules/puterai/OpenAIImageGenerationService.js +++ b/src/backend/src/modules/puterai/OpenAIImageGenerationService.js @@ -1,11 +1,26 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const BaseService = require("../../services/BaseService"); const { TypedValue } = require("../../services/drivers/meta/Runtime"); const { Context } = require("../../util/context"); + +/** +* Service class for generating images using OpenAI's DALL-E API. +* Extends BaseService to provide image generation capabilities through +* the puter-image-generation interface. Supports different aspect ratios +* (square, portrait, landscape) and handles API authentication, request +* validation, and spending tracking. +*/ class OpenAIImageGenerationService extends BaseService { static MODULES = { openai: require('openai'), } + /** + * Initializes the OpenAI client with API credentials from config + * @private + * @async + * @returns {Promise} + */ async _init () { const sk_key = this.config?.openai?.secret_key ?? @@ -24,6 +39,15 @@ class OpenAIImageGenerationService extends BaseService { } }, ['puter-image-generation']: { + /** + * Generates an image using OpenAI's DALL-E API + * @param {string} prompt - The text description of the image to generate + * @param {Object} options - Generation options + * @param {Object} options.ratio - Image dimensions ratio object with w/h properties + * @param {string} [options.model='dall-e-3'] - The model to use for generation + * @returns {Promise} URL of the generated image + * @throws {Error} If prompt is not a string or ratio is invalid + */ async generate ({ prompt, test_mode }) { if ( test_mode ) { return new TypedValue({ diff --git a/src/backend/src/modules/puterai/PuterAIModule.js b/src/backend/src/modules/puterai/PuterAIModule.js index 8844592df9..ddc8358805 100644 --- a/src/backend/src/modules/puterai/PuterAIModule.js +++ b/src/backend/src/modules/puterai/PuterAIModule.js @@ -1,7 +1,23 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const { AdvancedBase } = require("@heyputer/putility"); const config = require("../../config"); + +/** +* PuterAIModule class extends AdvancedBase to manage and register various AI services. +* This module handles the initialization and registration of multiple AI-related services +* including text processing, speech synthesis, chat completion, and image generation. +* Services are conditionally registered based on configuration settings, allowing for +* flexible deployment with different AI providers like AWS, OpenAI, Claude, Together AI, +* Mistral, Groq, and XAI. +* @extends AdvancedBase +*/ class PuterAIModule extends AdvancedBase { + /** + * Module for managing AI-related services in the Puter platform + * Extends AdvancedBase to provide core functionality + * Handles registration and configuration of various AI services like OpenAI, Claude, AWS services etc. + */ async install (context) { const services = context.get('services'); diff --git a/src/backend/src/modules/puterai/README.md b/src/backend/src/modules/puterai/README.md new file mode 100644 index 0000000000..6e04881615 --- /dev/null +++ b/src/backend/src/modules/puterai/README.md @@ -0,0 +1,333 @@ +# PuterAIModule + +PuterAIModule class extends AdvancedBase to manage and register various AI services. +This module handles the initialization and registration of multiple AI-related services +including text processing, speech synthesis, chat completion, and image generation. +Services are conditionally registered based on configuration settings, allowing for +flexible deployment with different AI providers like AWS, OpenAI, Claude, Together AI, +Mistral, Groq, and XAI. + +## Services + +### AIChatService + +AIChatService class extends BaseService to provide AI chat completion functionality. +Manages multiple AI providers, models, and fallback mechanisms for chat interactions. +Handles model registration, usage tracking, cost calculation, content moderation, +and implements the puter-chat-completion driver interface. Supports streaming responses +and maintains detailed model information including pricing and capabilities. + +#### Listeners + +##### `boot.consolidation` + +Handles consolidation during service boot by registering service aliases +and populating model lists/maps from providers. + +Registers each provider as an 'ai-chat' service alias and fetches their +available models and pricing information. Populates: +- simple_model_list: Basic list of supported models +- detail_model_list: Detailed model info including costs +- detail_model_map: Maps model IDs/aliases to their details + +#### Methods + +##### `register_provider` + + + +##### `moderate` + +Moderates chat messages for inappropriate content using OpenAI's moderation service + +###### Parameters + +- **params:** The parameters object +- **params.messages:** Array of chat messages to moderate + +##### `get_delegate` + +Gets the appropriate delegate service for handling chat completion requests. +If the intended service is this service (ai-chat), returns undefined. +Otherwise returns the intended service wrapped as a puter-chat-completion interface. + +##### `get_fallback_model` + +Find an appropriate fallback model by sorting the list of models +by the euclidean distance of the input/output prices and selecting +the first one that is not in the tried list. + +###### Parameters + +- **param0:** null + +##### `get_model_from_request` + + + +### AIInterfaceService + +Service class that manages AI interface registrations and configurations. +Handles registration of various AI services including OCR, chat completion, +image generation, and text-to-speech interfaces. Each interface defines +its available methods, parameters, and expected results. + +#### Listeners + +##### `driver.register.interfaces` + +Service class for managing AI interface registrations and configurations. +Extends the base service to provide AI-related interface management. +Handles registration of OCR, chat completion, image generation, and TTS interfaces. + +### AITestModeService + +Service class that handles AI test mode functionality. +Extends BaseService to register test services for AI chat completions. +Used for testing and development of AI-related features by providing +a mock implementation of the chat completion service. + +### AWSPollyService + +AWSPollyService class provides text-to-speech functionality using Amazon Polly. +Extends BaseService to integrate with AWS Polly for voice synthesis operations. +Implements voice listing, speech synthesis, and voice selection based on language. +Includes caching for voice descriptions and supports both text and SSML inputs. + +#### Methods + +##### `describe_voices` + +Describes available AWS Polly voices and caches the results + +##### `synthesize_speech` + +Synthesizes speech from text using AWS Polly + +###### Parameters + +- **text:** The text to synthesize +- **options:** Synthesis options +- **options.format:** Output audio format (e.g. 'mp3') + +### AWSTextractService + +AWSTextractService class - Provides OCR (Optical Character Recognition) functionality using AWS Textract +Extends BaseService to integrate with AWS Textract for document analysis and text extraction. +Implements driver capabilities and puter-ocr interface for document recognition. +Handles both S3-stored and buffer-based document processing with automatic region management. + +#### Methods + +##### `analyze_document` + +Analyzes a document using AWS Textract to extract text and layout information + +###### Parameters + +- **file_facade:** Interface to access the document file + +### ClaudeEnoughService + +ClaudeEnoughService - A service class that implements a Claude-like AI interface +Extends XAIService to provide Claude-compatible responses while using alternative AI models. +Includes custom system prompts and model adaptation to simulate Claude's behavior +in the Puter platform's chat completion interface. + +#### Methods + +##### `get_system_prompt` + +Service that emulates Claude's behavior using alternative AI models + +##### `adapt_model` + + + +### ClaudeService + +ClaudeService class extends BaseService to provide integration with Anthropic's Claude AI models. +Implements the puter-chat-completion interface for handling AI chat interactions. +Manages message streaming, token limits, model selection, and API communication with Claude. +Supports system prompts, message adaptation, and usage tracking. + +#### Methods + +##### `get_default_model` + +Returns the default model identifier for Claude API interactions + +### FakeChatService + +FakeChatService - A mock implementation of a chat service that extends BaseService. +Provides fake chat completion responses using Lorem Ipsum text generation. +Used for testing and development purposes when a real chat service is not needed. +Implements the 'puter-chat-completion' interface with list() and complete() methods. + +### GroqAIService + +Service class for integrating with Groq AI's language models. +Extends BaseService to provide chat completion capabilities through the Groq API. +Implements the puter-chat-completion interface for model management and text generation. +Supports both streaming and non-streaming responses, handles multiple models including +various versions of Llama, Mixtral, and Gemma, and manages usage tracking. + +#### Methods + +##### `get_default_model` + +Returns the default model ID for the Groq AI service + +### MistralAIService + +MistralAIService class extends BaseService to provide integration with the Mistral AI API. +Implements chat completion functionality with support for various Mistral models including +mistral-large, pixtral, codestral, and ministral variants. Handles both streaming and +non-streaming responses, token usage tracking, and model management. Provides cost information +for different models and implements the puter-chat-completion interface. + +#### Methods + +##### `get_default_model` + +Populates the internal models array with available Mistral AI models and their metadata +Fetches model data from the API, filters based on cost configuration, and stores +model objects containing ID, name, aliases, context length, capabilities, and pricing + +### OpenAICompletionService + +OpenAICompletionService class provides an interface to OpenAI's chat completion API. +Extends BaseService to handle chat completions, message moderation, token counting, +and streaming responses. Implements the puter-chat-completion interface and manages +OpenAI API interactions with support for multiple models including GPT-4 variants. +Handles usage tracking, spending records, and content moderation. + +#### Methods + +##### `get_default_model` + +Gets the default model identifier for OpenAI completions + +##### `check_moderation` + +Checks text content against OpenAI's moderation API for inappropriate content + +###### Parameters + +- **text:** The text content to check for moderation + +##### `complete` + +Completes a chat conversation using OpenAI's API + +###### Parameters + +- **messages:** Array of message objects or strings representing the conversation +- **options:** Configuration options +- **options.stream:** Whether to stream the response +- **options.moderation:** Whether to perform content moderation +- **options.model:** The model to use for completion + +### OpenAIImageGenerationService + +Service class for generating images using OpenAI's DALL-E API. +Extends BaseService to provide image generation capabilities through +the puter-image-generation interface. Supports different aspect ratios +(square, portrait, landscape) and handles API authentication, request +validation, and spending tracking. + +#### Methods + +##### `generate` + + + +### TogetherAIService + +TogetherAIService class provides integration with Together AI's language models. +Extends BaseService to implement chat completion functionality through the +puter-chat-completion interface. Manages model listings, chat completions, +and streaming responses while handling usage tracking and model fallback testing. + +#### Methods + +##### `get_default_model` + +Returns the default model ID for the Together AI service + +### XAIService + +XAIService class - Provides integration with X.AI's API for chat completions +Extends BaseService to implement the puter-chat-completion interface. +Handles model management, message adaptation, streaming responses, +and usage tracking for X.AI's language models like Grok. + +#### Methods + +##### `get_system_prompt` + +Gets the system prompt used for AI interactions + +##### `adapt_model` + + + +##### `get_default_model` + +Returns the default model identifier for the XAI service + +## Notes + +### Outside Imports + +This module has external relative imports. When these are +removed it may become possible to move this module to an +extension. + +**Imports:** +- `../../api/APIError` +- `../../services/auth/PermissionService` +- `../../services/BaseService` (use.BaseService) +- `../../services/database/consts` +- `../../services/drivers/meta/Construct` +- `../../services/drivers/meta/Runtime` +- `../../util/context` +- `../../services/BaseService` (use.BaseService) +- `../../services/BaseService` (use.BaseService) +- `../../services/BaseService` (use.BaseService) +- `../../services/drivers/meta/Runtime` +- `../../services/BaseService` (use.BaseService) +- `../../api/APIError` +- `../../services/BaseService` (use.BaseService) +- `../../util/langutil` +- `../../services/drivers/meta/Runtime` +- `../../api/APIError` +- `../../util/promise` +- `../../services/BaseService` (use.BaseService) +- `../../services/BaseService` (use.BaseService) +- `../../services/drivers/meta/Runtime` +- `../../util/langutil` +- `../../util/promise` +- `../../services/BaseService` (use.BaseService) +- `../../services/drivers/meta/Runtime` +- `../../util/langutil` +- `../../util/promise` +- `../../api/APIError` +- `../../services/BaseService` (use.BaseService) +- `../../services/drivers/meta/Runtime` +- `../../util/context` +- `../../util/smolutil` +- `../../util/langutil` +- `../../util/promise` +- `../../services/BaseService` (use.BaseService) +- `../../services/drivers/meta/Runtime` +- `../../util/context` +- `../../config` +- `../../services/BaseService` (use.BaseService) +- `../../services/drivers/meta/Runtime` +- `../../util/langutil` +- `../../util/promise` +- `../../services/BaseService` (use.BaseService) +- `../../util/langutil` +- `../../services/drivers/meta/Runtime` +- `../../util/promise` diff --git a/src/backend/src/modules/puterai/TogetherAIService.js b/src/backend/src/modules/puterai/TogetherAIService.js index 81774fe494..6ca2967b8d 100644 --- a/src/backend/src/modules/puterai/TogetherAIService.js +++ b/src/backend/src/modules/puterai/TogetherAIService.js @@ -1,9 +1,18 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const { PassThrough } = require("stream"); const BaseService = require("../../services/BaseService"); const { TypedValue } = require("../../services/drivers/meta/Runtime"); const { nou } = require("../../util/langutil"); -const { TeePromise } = require("../../util/promise"); +const { TeePromise } = require('@heyputer/putility').libs.promise; + +/** +* TogetherAIService class provides integration with Together AI's language models. +* Extends BaseService to implement chat completion functionality through the +* puter-chat-completion interface. Manages model listings, chat completions, +* and streaming responses while handling usage tracking and model fallback testing. +* @extends BaseService +*/ class TogetherAIService extends BaseService { static MODULES = { ['together-ai']: require('together-ai'), @@ -11,6 +20,13 @@ class TogetherAIService extends BaseService { uuidv4: require('uuid').v4, } + + /** + * Initializes the TogetherAI service by setting up the API client and registering as a chat provider + * @async + * @returns {Promise} + * @private + */ async _init () { const require = this.require; const Together = require('together-ai'); @@ -27,20 +43,41 @@ class TogetherAIService extends BaseService { }); } + + /** + * Returns the default model ID for the Together AI service + * @returns {string} The ID of the default model (meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo) + */ get_default_model () { return 'meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo'; } static IMPLEMENTS = { ['puter-chat-completion']: { + /** + * Implements the puter-chat-completion interface for TogetherAI service + * Contains methods for listing models and generating chat completions + * @property {Object} models - Method to get available models + * @property {Object} list - Method to get list of model IDs + * @property {Object} complete - Method to generate chat completions + */ async models () { return await this.models_(); }, + /** + * Retrieves available AI models from the Together API + * @returns {Promise} Array of model objects with their properties + * @implements {puter-chat-completion.models} + */ async list () { let models = this.modules.kv.get(`${this.kvkey}:models`); if ( ! models ) models = await this.models_(); return models.map(model => model.id); }, + /** + * Lists available AI model IDs from the cache or fetches them if not cached + * @returns {Promise} Array of model ID strings + */ async complete ({ messages, stream, model }) { if ( model === 'model-fallback-test-1' ) { throw new Error('Model Fallback Test 1'); @@ -103,6 +140,14 @@ class TogetherAIService extends BaseService { } } + + /** + * Fetches and caches available AI models from Together API + * @private + * @returns {Promise} Array of model objects containing id, name, context length, + * description and pricing information + * @remarks Models are cached for 5 minutes in KV store + */ async models_ () { let models = this.modules.kv.get(`${this.kvkey}:models`); if ( models ) return models; diff --git a/src/backend/src/modules/puterai/XAIService.js b/src/backend/src/modules/puterai/XAIService.js index 8225ffb14f..58e5c92904 100644 --- a/src/backend/src/modules/puterai/XAIService.js +++ b/src/backend/src/modules/puterai/XAIService.js @@ -1,9 +1,10 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const { default: Anthropic } = require("@anthropic-ai/sdk"); const BaseService = require("../../services/BaseService"); const { whatis, nou } = require("../../util/langutil"); const { PassThrough } = require("stream"); const { TypedValue } = require("../../services/drivers/meta/Runtime"); -const { TeePromise } = require("../../util/promise"); +const { TeePromise } = require('@heyputer/putility').libs.promise; const PUTER_PROMPT = ` You are running on an open-source platform called Puter, @@ -14,11 +15,24 @@ const PUTER_PROMPT = ` user of the driver interface (typically an app on Puter): `.replace('\n', ' ').trim(); + +/** +* XAIService class - Provides integration with X.AI's API for chat completions +* Extends BaseService to implement the puter-chat-completion interface. +* Handles model management, message adaptation, streaming responses, +* and usage tracking for X.AI's language models like Grok. +* @extends BaseService +*/ class XAIService extends BaseService { static MODULES = { openai: require('openai'), } + + /** + * Gets the system prompt used for AI interactions + * @returns {string} The base system prompt that identifies the AI as running on Puter + */ get_system_prompt () { return PUTER_PROMPT; } @@ -27,6 +41,12 @@ class XAIService extends BaseService { return model; } + + /** + * Initializes the XAI service by setting up the OpenAI client and registering with the AI chat provider + * @private + * @returns {Promise} Resolves when initialization is complete + */ async _init () { this.openai = new this.modules.openai.OpenAI({ apiKey: this.global_config.services.xai.apiKey, @@ -40,15 +60,30 @@ class XAIService extends BaseService { }); } + + /** + * Returns the default model identifier for the XAI service + * @returns {string} The default model ID 'grok-beta' + */ get_default_model () { return 'grok-beta'; } static IMPLEMENTS = { ['puter-chat-completion']: { + /** + * Implements the interface for the puter-chat-completion driver + * Contains methods for listing models, getting model details, + * and handling chat completions with streaming support + * @type {Object} + */ async models () { return await this.models_(); }, + /** + * Returns a list of available AI models with their capabilities and pricing details + * @returns {Promise} Array of model objects containing id, name, context window size, and cost information + */ async list () { const models = await this.models_(); const model_names = []; @@ -60,6 +95,10 @@ class XAIService extends BaseService { } return model_names; }, + /** + * Returns a list of all available model names including their aliases + * @returns {Promise} Array of model names and their aliases + */ async complete ({ messages, stream, model }) { model = this.adapt_model(model); const adapted_messages = []; @@ -162,6 +201,16 @@ class XAIService extends BaseService { } } + + /** + * Retrieves available AI models and their specifications + * @returns {Promise} Array of model objects containing: + * - id: Model identifier string + * - name: Human readable model name + * - context: Maximum context window size + * - cost: Pricing information object with currency and rates + * @private + */ async models_ () { return [ { diff --git a/src/backend/src/modules/puterai/lib/AsModeration.js b/src/backend/src/modules/puterai/lib/AsModeration.js new file mode 100644 index 0000000000..2f20c76d48 --- /dev/null +++ b/src/backend/src/modules/puterai/lib/AsModeration.js @@ -0,0 +1,77 @@ +const { default: dedent } = require("dedent"); + +class AsModeration { + constructor ({ chat, model }) { + this.chat = chat; + this.model = model; + } + + async moderate (text) { + // copied from Anthropic documentation + // https://docs.anthropic.com/en/docs/about-claude/use-case-guides/content-moderation + const unsafe_categories = [ + 'Child Exploitation', + 'Conspiracy Theories', + 'Hate', + 'Indiscriminate Weapons', + 'Intellectual Property', + 'Non-Violent Crimes', + 'Privacy', + 'Self-Harm', + 'Sex Crimes', + 'Sexual Content', + 'Specialized Advice', + 'Violent Crimes' + ]; + + const unsafe_category_str = unsafe_categories.join('\n'); + + const assessment_prompt = dedent(` + Determine whether the following message warrants moderation, + based on the unsafe categories outlined below. + + Message: + + ${text + .replace('<', '<') + .replace('>', '>') + } + + + Unsafe Categories: + + ${unsafe_category_str} + + + Respond with ONLY a JSON object, using the format below: + {{ + "violation": , + "categories": [Comma-separated list of violated categories], + "explanation": [Optional. Only include if there is a violation.] + }} + `); + + const result = await this.chat.complete({ + messages: [ + { + role: 'user', + content: assessment_prompt, + } + ] + }); + + console.log('result???', require('util').inspect(result, { depth: null })); + + const str = result.message?.content?.[0]?.text ?? + result.messages?.[0]?.content?.[0]?.text ?? + '{ "violation": true }'; + + const parsed = JSON.parse(str); + console.log('parsed?', parsed); + return ! parsed.violation; + } +} + +module.exports = { + AsModeration, +}; diff --git a/src/backend/src/filesystem/storage/DatabaseFSEntryFetcher.js b/src/backend/src/modules/puterfs/DatabaseFSEntryFetcher.js similarity index 94% rename from src/backend/src/filesystem/storage/DatabaseFSEntryFetcher.js rename to src/backend/src/modules/puterfs/DatabaseFSEntryFetcher.js index 82ece7eb2e..b44bfcd145 100644 --- a/src/backend/src/filesystem/storage/DatabaseFSEntryFetcher.js +++ b/src/backend/src/modules/puterfs/DatabaseFSEntryFetcher.js @@ -17,16 +17,11 @@ * along with this program. If not, see . */ const { DB_READ } = require("../../services/database/consts"); -const { abtest } = require("../../util/otelutil"); -const { NodePathSelector, NodeUIDSelector, NodeInternalIDSelector, NodeChildSelector, RootNodeSelector } = require("../node/selectors"); - -module.exports = class DatabaseFSEntryFetcher { - constructor ({ services }) { - this.services = services; - this.log = services.get('log-service').create('DatabaseFSEntryFetcher'); - - this.db = services.get('database').get(DB_READ, 'filesystem'); +const { NodePathSelector, NodeUIDSelector, NodeInternalIDSelector, NodeChildSelector, RootNodeSelector } = require("../../filesystem/node/selectors"); +const BaseService = require("../../services/BaseService"); +module.exports = class DatabaseFSEntryFetcher extends BaseService { + _construct () { this.defaultProperties = [ 'id', 'associated_app_id', @@ -57,6 +52,10 @@ module.exports = class DatabaseFSEntryFetcher { ] } + _init () { + this.db = this.services.get('database').get(DB_READ, 'filesystem'); + } + async find (selector, fetch_entry_options) { if ( selector instanceof RootNodeSelector ) { return selector.entry; diff --git a/src/backend/src/filesystem/storage/DatabaseFSEntryService.js b/src/backend/src/modules/puterfs/DatabaseFSEntryService.js similarity index 94% rename from src/backend/src/filesystem/storage/DatabaseFSEntryService.js rename to src/backend/src/modules/puterfs/DatabaseFSEntryService.js index ecf674ef23..21f6541790 100644 --- a/src/backend/src/filesystem/storage/DatabaseFSEntryService.js +++ b/src/backend/src/modules/puterfs/DatabaseFSEntryService.js @@ -19,11 +19,12 @@ const { AdvancedBase } = require("@heyputer/putility"); const { id2path } = require("../../helpers"); -const { PuterPath } = require("../lib/PuterPath"); -const { NodeUIDSelector } = require("../node/selectors"); +const { PuterPath } = require("../../filesystem/lib/PuterPath"); +const { NodeUIDSelector } = require("../../filesystem/node/selectors"); const { OtelFeature } = require("../../traits/OtelFeature"); const { Context } = require("../../util/context"); const { DB_WRITE } = require("../../services/database/consts"); +const BaseService = require("../../services/BaseService"); class AbstractDatabaseFSEntryOperation { static STATUS_PENDING = {}; @@ -191,7 +192,7 @@ class DatabaseFSEntryDelete extends AbstractDatabaseFSEntryOperation { } -class DatabaseFSEntryService extends AdvancedBase { +class DatabaseFSEntryService extends BaseService { static STATUS_READY = {}; static STATUS_RUNNING_JOB = {}; @@ -211,23 +212,7 @@ class DatabaseFSEntryService extends AdvancedBase { ]), ] - constructor ({ services, label }) { - super(); - this.db = services.get('database').get(DB_WRITE, 'filesystem'); - - this.log = services.get('log-service').create('fsentry-service'); - - this.label = label || 'DatabaseFSEntryService'; - - const params = services.get('params'); - params.createParameters('fsentry-service', [ - { - id: 'max_queue', - description: 'Maximum queue size', - default: 50, - }, - ], this); - + _construct () { this.status = this.constructor.STATUS_READY; this.currentState = { @@ -242,9 +227,22 @@ class DatabaseFSEntryService extends AdvancedBase { this.entryListeners_ = {}; this.mkPromiseForQueueSize_(); + } + + _init () { + const params = this.services.get('params'); + params.createParameters('fsentry-service', [ + { + id: 'max_queue', + description: 'Maximum queue size', + default: 50, + }, + ], this); + + this.db = this.services.get('database').get(DB_WRITE, 'filesystem'); // Register information providers - const info = services.get('information'); + const info = this.services.get('information'); // uuid -> path via mysql info.given('fs.fsentry:uuid').provide('fs.fsentry:path') @@ -256,13 +254,10 @@ class DatabaseFSEntryService extends AdvancedBase { return '/-void/' + uuid; } }); - - (async () => { - await services.ready; - if ( services.has('commands') ) { - this._registerCommands(services.get('commands')); - } - })(); + } + + ['__on_boot.consolidation'] () { + this._registerCommands(this.services.get('commands')); } mkPromiseForQueueSize_ () { @@ -462,7 +457,6 @@ class DatabaseFSEntryService extends AdvancedBase { const queue = this.currentState.queue; this.log.info( - `\x1B[36;1m[${this.label}]\x1B[0m ` + `Executing ${queue.length} operations...` ); @@ -502,7 +496,6 @@ class DatabaseFSEntryService extends AdvancedBase { this.status = this.constructor.STATUS_READY; this.log.info( - `\x1B[36;1m[${this.label}]\x1B[0m ` + `Finished ${queue.length} operations.` ) diff --git a/src/backend/src/services/MountpointService.js b/src/backend/src/modules/puterfs/MountpointService.js similarity index 65% rename from src/backend/src/services/MountpointService.js rename to src/backend/src/modules/puterfs/MountpointService.js index 2b49426db0..6505968b2c 100644 --- a/src/backend/src/services/MountpointService.js +++ b/src/backend/src/modules/puterfs/MountpointService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"claude"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -18,7 +19,7 @@ */ // const Mountpoint = o => ({ ...o }); -const BaseService = require("./BaseService"); +const BaseService = require("../../services/BaseService"); /** * This will eventually be a service which manages the storage @@ -28,7 +29,22 @@ const BaseService = require("./BaseService"); * in situations where ContextInitService isn't able to * initialize a context. */ +/** +* @class MountpointService +* @extends BaseService +* @description Service class responsible for managing storage backends for mountpoints. +* Currently provides a temporary solution for accessing storage backend when context +* initialization is not possible. Will be expanded to handle multiple mountpoints +* and their associated storage backends in future implementations. +*/ class MountpointService extends BaseService { + /** + * Initializes the MountpointService instance + * Sets up initial state with null storage backend + * @private + * @async + * @returns {Promise} + */ async _init () { // this.mountpoints_ = {}; @@ -40,6 +56,10 @@ class MountpointService extends BaseService { set_storage (storage) { this.storage_ = storage; } + /** + * Gets the current storage backend instance + * @returns {Object} The storage backend instance + */ get_storage () { return this.storage_; } diff --git a/src/backend/src/modules/puterfs/PuterFSModule.js b/src/backend/src/modules/puterfs/PuterFSModule.js new file mode 100644 index 0000000000..c78220478f --- /dev/null +++ b/src/backend/src/modules/puterfs/PuterFSModule.js @@ -0,0 +1,24 @@ +const { AdvancedBase } = require("@heyputer/putility"); + +class PuterFSModule extends AdvancedBase { + async install (context) { + const services = context.get('services'); + + const { ResourceService } = require("./ResourceService"); + services.registerService('resourceService', ResourceService); + + const { DatabaseFSEntryService } = require("./DatabaseFSEntryService"); + services.registerService('fsEntryService', DatabaseFSEntryService); + + const { SizeService } = require('./SizeService'); + services.registerService('sizeService', SizeService); + + const { MountpointService } = require('./MountpointService'); + services.registerService('mountpoint', MountpointService); + + const DatabaseFSEntryFetcher = require("./DatabaseFSEntryFetcher"); + services.registerService('fsEntryFetcher', DatabaseFSEntryFetcher); + } +} + +module.exports = { PuterFSModule }; diff --git a/src/backend/src/filesystem/storage/ResourceService.js b/src/backend/src/modules/puterfs/ResourceService.js similarity index 94% rename from src/backend/src/filesystem/storage/ResourceService.js rename to src/backend/src/modules/puterfs/ResourceService.js index 64bfca4917..f1d0bba67d 100644 --- a/src/backend/src/filesystem/storage/ResourceService.js +++ b/src/backend/src/modules/puterfs/ResourceService.js @@ -16,13 +16,13 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -let waiti = 0; +const BaseService = require("../../services/BaseService"); const { NodePathSelector, NodeUIDSelector, NodeInternalIDSelector, NodeChildSelector, -} = require("../node/selectors"); +} = require("../../filesystem/node/selectors"); const RESOURCE_STATUS_PENDING_CREATE = {}; const RESOURCE_STATUS_PENDING_UPDATE = {}; @@ -41,13 +41,11 @@ const RS_DIRECTORY_PENDING_CHILD_INSERT = {}; * At least for now; I'm sure we'll think of a smarter way to * handle this in the future. */ -class ResourceService { - constructor ({ services }) { +class ResourceService extends BaseService { + _construct () { this.uidToEntry = {}; this.uidToPath = {}; this.pathToEntry = {}; - - this.log = services.get('log-service').create('resource-service'); } register (entry) { @@ -103,7 +101,6 @@ class ResourceService { } async waitForResource (selector) { - const i = waiti++; if ( selector instanceof NodePathSelector ) { await this.waitForResourceByPath(selector.value); } diff --git a/src/backend/src/filesystem/storage/SizeService.js b/src/backend/src/modules/puterfs/SizeService.js similarity index 91% rename from src/backend/src/filesystem/storage/SizeService.js rename to src/backend/src/modules/puterfs/SizeService.js index 3d9929d039..d4649048df 100644 --- a/src/backend/src/filesystem/storage/SizeService.js +++ b/src/backend/src/modules/puterfs/SizeService.js @@ -17,9 +17,11 @@ * along with this program. If not, see . */ const { get_dir_size, id2path, get_user, invalidate_cached_user_by_id } = require("../../helpers"); +const BaseService = require("../../services/BaseService"); const { DB_WRITE } = require("../../services/database/consts"); const { Context } = require("../../util/context"); +const { nou } = require("../../util/langutil"); // TODO: expose to a utility library class UserParameter { @@ -32,15 +34,18 @@ class UserParameter { } } -class SizeService { - constructor ({ services }) { - this.db = services.get('database').get(DB_WRITE, 'filesystem'); - this.log = services.get('log-service').create('size-service'); - this.errors = services.get('error-service').create(this.log); - +class SizeService extends BaseService { + _construct () { this.usages = {}; + } + + _init () { + this.db = this.services.get('database').get(DB_WRITE, 'filesystem'); + + } - const svc_commands = services.get('commands'); + ['__on_boot.consolidate'] () { + const svc_commands = this.services.get('commands'); svc_commands.registerCommands('size', [ { id: 'get-usage', @@ -118,6 +123,14 @@ class SizeService { async get_storage_capacity (user_or_id) { const user = await UserParameter.adapt(user_or_id); + if ( ! this.global_config.is_storage_limited ) { + return this.global_config.available_device_storage; + } + + if ( nou(user.free_storage) ) { + return this.global_config.storage_capacity; + } + return user.free_storage; } diff --git a/src/backend/src/modules/selfhosted/ComplainAboutVersionsService.js b/src/backend/src/modules/selfhosted/ComplainAboutVersionsService.js index 9effd10d0b..087d2bf087 100644 --- a/src/backend/src/modules/selfhosted/ComplainAboutVersionsService.js +++ b/src/backend/src/modules/selfhosted/ComplainAboutVersionsService.js @@ -61,7 +61,6 @@ class ComplainAboutVersionsService extends BaseService { let timeago = (() => { let years = cur_date_obj.getFullYear() - eol_date.getFullYear(); let months = cur_date_obj.getMonth() - eol_date.getMonth(); - let days = cur_date_obj.getDate() - eol_date.getDate(); let str = ''; while ( years > 0 ) { diff --git a/src/backend/src/modules/selfhosted/DefaultUserService.js b/src/backend/src/modules/selfhosted/DefaultUserService.js index db1c5d087b..774d6cc788 100644 --- a/src/backend/src/modules/selfhosted/DefaultUserService.js +++ b/src/backend/src/modules/selfhosted/DefaultUserService.js @@ -20,14 +20,14 @@ const { QuickMkdir } = require("../../filesystem/hl_operations/hl_mkdir"); const { HLWrite } = require("../../filesystem/hl_operations/hl_write"); const { NodePathSelector } = require("../../filesystem/node/selectors"); const { surrounding_box } = require("../../fun/dev-console-ui-utils"); -const { get_user, generate_system_fsentries, invalidate_cached_user } = require("../../helpers"); +const { get_user, invalidate_cached_user } = require("../../helpers"); const { Context } = require("../../util/context"); -const { asyncSafeSetInterval } = require("../../util/promise"); +const { asyncSafeSetInterval } = require('@heyputer/putility').libs.promise; const { buffer_to_stream } = require("../../util/streamutil"); const BaseService = require("../../services/BaseService"); const { Actor, UserActorType } = require("../../services/auth/Actor"); const { DB_WRITE } = require("../../services/database/consts"); -const { quot } = require("../../util/strutil"); +const { quot } = require('@heyputer/putility').libs.string; const USERNAME = 'admin'; @@ -165,7 +165,8 @@ class DefaultUserService extends BaseService { ], ); user.password = password_hashed; - await generate_system_fsentries(user); + const svc_user = this.services.get('user'); + await svc_user.generate_default_fsentries({ user }); // generate default files for admin user const svc_fs = this.services.get('filesystem'); const make_tree_ = async ({ components, tree }) => { diff --git a/src/backend/src/modules/selfhosted/SelfHostedModule.js b/src/backend/src/modules/selfhosted/SelfHostedModule.js index a10989428e..f49ed4149d 100644 --- a/src/backend/src/modules/selfhosted/SelfHostedModule.js +++ b/src/backend/src/modules/selfhosted/SelfHostedModule.js @@ -34,6 +34,9 @@ class SelfHostedModule extends AdvancedBase { const DevWatcherService = require('./DevWatcherService'); const path_ = require('path'); + + const { DBKVService } = require("../../services/DBKVService"); + services.registerService('puter-kvstore', DBKVService); // TODO: sucks const RELATIVE_PATH = '../../../../../'; diff --git a/src/backend/src/modules/selfhosted/SelfhostedService.js b/src/backend/src/modules/selfhosted/SelfhostedService.js index 7a0109eb33..e421920ccf 100644 --- a/src/backend/src/modules/selfhosted/SelfhostedService.js +++ b/src/backend/src/modules/selfhosted/SelfhostedService.js @@ -16,9 +16,6 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -const { DBKVStore } = require("../../drivers/DBKVStore"); -const { EntityStoreImplementation } = require("../../drivers/EntityStoreImplementation"); -const { HelloWorld } = require("../../drivers/HelloWorld"); const BaseService = require("../../services/BaseService"); class SelfhostedService extends BaseService { @@ -27,12 +24,6 @@ class SelfhostedService extends BaseService { ` async _init () { - const svc_driver = this.services.get('driver'); - - svc_driver.register_driver('puter-kvstore', new DBKVStore()); - svc_driver.register_driver('puter-apps', new EntityStoreImplementation({ service: 'es:app' })); - svc_driver.register_driver('puter-subdomains', new EntityStoreImplementation({ service: 'es:subdomain' })); - svc_driver.register_driver('puter-notifications', new EntityStoreImplementation({ service: 'es:notification' })); } } diff --git a/src/backend/src/modules/template/README.md b/src/backend/src/modules/template/README.md new file mode 100644 index 0000000000..e272a18fe0 --- /dev/null +++ b/src/backend/src/modules/template/README.md @@ -0,0 +1,56 @@ +# TemplateModule + +This is a template module that you can copy and paste to create new modules. + +This module is also included in `EssentialModules`, which means it will load +when Puter boots. If you're just testing something, you can add it here +temporarily. + +## Services + +### TemplateService + +This is a template service that you can copy and paste to create new services. +You can also add to this service temporarily to test something. + +#### Listeners + +##### `install.routes` + +TemplateService listens to this event to provide an example endpoint + +##### `boot.consolidation` + +TemplateService listens to this event to provide an example event + +##### `boot.activation` + +TemplateService listens to this event to show you that it's here + +##### `start.webserver` + +TemplateService listens to this event to show you that it's here + +## Libraries + +### hello_world + +#### Functions + +##### `hello_world` + +This is a simple function that returns a string. +You can probably guess what string it returns. + +## Notes + +### Outside Imports + +This module has external relative imports. When these are +removed it may become possible to move this module to an +extension. + +**Imports:** +- `../../util/context.js` +- `../../services/BaseService` (use.BaseService) +- `../../util/expressutil` diff --git a/src/backend/src/modules/template/TemplateModule.js b/src/backend/src/modules/template/TemplateModule.js new file mode 100644 index 0000000000..5e22fd36c9 --- /dev/null +++ b/src/backend/src/modules/template/TemplateModule.js @@ -0,0 +1,53 @@ +/* + * Copyright (C) 2024 Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +const { AdvancedBase } = require("@heyputer/putility"); + +/** + * This is a template module that you can copy and paste to create new modules. + * + * This module is also included in `EssentialModules`, which means it will load + * when Puter boots. If you're just testing something, you can add it here + * temporarily. + */ +class TemplateModule extends AdvancedBase { + async install (context) { + // === LIBS === // + const useapi = context.get('useapi'); + + const lib = require('./lib/__lib__.js'); + + // In extensions: use('workinprogress').hello_world(); + // In services classes: see TemplateService.js + useapi.def(`workinprogress`, lib, { assign: true }); + + useapi.def('core.context', require('../../util/context.js').Context); + + // === SERVICES === // + const services = context.get('services'); + + const { TemplateService } = require('./TemplateService.js'); + services.registerService('template-service', TemplateService); + } + +} + +module.exports = { + TemplateModule +}; diff --git a/src/backend/src/modules/template/TemplateService.js b/src/backend/src/modules/template/TemplateService.js new file mode 100644 index 0000000000..e079ac2c4f --- /dev/null +++ b/src/backend/src/modules/template/TemplateService.js @@ -0,0 +1,99 @@ +/* + * Copyright (C) 2024 Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TODO: import via `USE` static member +const BaseService = require("../../services/BaseService"); +const { Endpoint } = require("../../util/expressutil"); + +/** + * This is a template service that you can copy and paste to create new services. + * You can also add to this service temporarily to test something. + */ +class TemplateService extends BaseService { + static USE = { + // - Defined by lib/__lib__.js, + // - Exposed to `useapi` by TemplateModule.js + workinprogress: 'workinprogress' + } + + _construct () { + // Use this override to initialize instance variables. + } + + async _init () { + // This is where you initialize the service and prepare + // for the consolidation phase. + this.log.info("I am the template service."); + } + + /** + * TemplateService listens to this event to provide an example endpoint + */ + ['__on_install.routes'] (_, { app }) { + this.log.info("TemplateService get the event for installing endpoint."); + Endpoint({ + route: '/example-endpoint', + methods: ['GET'], + handler: async (req, res) => { + res.send(this.workinprogress.hello_world()); + } + }).attach(app); + // ^ Don't forget to attach the endpoint to the app! + // it's very easy to forget this step. + } + + /** + * TemplateService listens to this event to provide an example event + */ + ['__on_boot.consolidation'] () { + // At this stage, all services have been initialized and it is + // safe to start emitting events. + this.log.info("TemplateService sees consolidation boot phase."); + + const svc_event = this.services.get('event'); + + svc_event.on('template-service.hello', (_eventid, event_data) => { + this.log.info('template-service said hello to itself; this is expected', { + event_data, + }); + }); + + svc_event.emit('template-service.hello', { + message: 'Hello all you other services! I am the template service.' + }); + } + /** + * TemplateService listens to this event to show you that it's here + */ + ['__on_boot.activation'] () { + this.log.info("TemplateService sees activation boot phase."); + } + + /** + * TemplateService listens to this event to show you that it's here + */ + ['__on_start.webserver'] () { + this.log.info("TemplateService sees it's time to start web servers."); + } +} + +module.exports = { + TemplateService +}; + diff --git a/src/backend/src/modules/template/lib/__lib__.js b/src/backend/src/modules/template/lib/__lib__.js new file mode 100644 index 0000000000..51104b7ba9 --- /dev/null +++ b/src/backend/src/modules/template/lib/__lib__.js @@ -0,0 +1,3 @@ +module.exports = { + hello_world: require('./hello_world.js'), +}; diff --git a/src/backend/src/modules/template/lib/hello_world.js b/src/backend/src/modules/template/lib/hello_world.js new file mode 100644 index 0000000000..cbef6e3f27 --- /dev/null +++ b/src/backend/src/modules/template/lib/hello_world.js @@ -0,0 +1,10 @@ + +/** + * This is a simple function that returns a string. + * You can probably guess what string it returns. + */ +const hello_world = () => { + return "Hello, world!"; +} + +module.exports = hello_world; diff --git a/src/backend/src/modules/web/APIErrorService.js b/src/backend/src/modules/web/APIErrorService.js new file mode 100644 index 0000000000..14ba281c35 --- /dev/null +++ b/src/backend/src/modules/web/APIErrorService.js @@ -0,0 +1,54 @@ +const APIError = require("../../api/APIError"); +const BaseService = require("../../services/BaseService"); + +/** + * @typedef {Object} ErrorSpec + * @property {string} code - The error code + * @property {string} status - HTTP status code + * @property {function} message - A function that generates an error message + */ + +/** + * The APIErrorService class provides a mechanism for registering and managing + * error codes and messages which may be sent to clients. + * + * This allows for a single source-of-truth for error codes and messages that + * are used by multiple services. + */ +class APIErrorService extends BaseService { + _construct () { + this.codes = { + ...this.constructor.codes, + }; + } + + // Hardcoded error codes from before this service was created + static codes = APIError.codes; + + /** + * Registers API error codes. + * + * @param {Object.} codes - A map of error codes to error specifications + */ + register (codes) { + for ( const code in codes ) { + this.codes[code] = codes[code]; + } + } + + create (code, fields) { + const error_spec = this.codes[code]; + if ( ! error_spec ) { + return new APIError(500, 'Missing error message.', null, { + code, + }); + } + + return new APIError(error_spec.status, error_spec.message, null, { + ...fields, + code, + }); + } +} + +module.exports = APIErrorService; diff --git a/src/backend/src/modules/web/README.md b/src/backend/src/modules/web/README.md new file mode 100644 index 0000000000..d2eb27c363 --- /dev/null +++ b/src/backend/src/modules/web/README.md @@ -0,0 +1,67 @@ +# WebModule + +This module initializes a pre-configured web server and socket.io server. +The main service, WebServerService, emits 'install.routes' and provides +the server instance to the callback. + +## Services + +### SocketioService + +SocketioService provides a service for sending messages to clients. +socket.io is used behind the scenes. This service provides a simpler +interface for sending messages to rooms or socket ids. + +#### Listeners + +##### `install.socketio` + +Initializes socket.io + +###### Parameters + +- **server:** The server to attach socket.io to. + +### WebServerService + +This class, WebServerService, is responsible for starting and managing the Puter web server. +It initializes the Express app, sets up middlewares, routes, and handles authentication and web sockets. +It also validates the host header and IP addresses to prevent security vulnerabilities. + +#### Listeners + +##### `boot.consolidation` + +This method initializes the backend web server for Puter. It sets up the Express app, configures middleware, and starts the HTTP server. + +##### `boot.activation` + +Starts the web server and listens for incoming connections. +This method sets up the Express app, sets up middleware, and starts the server on the specified port. +It also sets up the Socket.io server for real-time communication. + +##### `start.webserver` + +This method starts the web server by listening on the specified port. It tries multiple ports if the first one is in use. +If the `config.http_port` is set to 'auto', it will try to find an available port in a range of 4100 to 4299. +Once the server is up and running, it emits the 'start.webserver' and 'ready.webserver' events. +If the `config.env` is set to 'dev' and `config.no_browser_launch` is false, it will open the Puter URL in the default browser. + +## Notes + +### Outside Imports + +This module has external relative imports. When these are +removed it may become possible to move this module to an +extension. + +**Imports:** +- `../../services/BaseService` (use.BaseService) +- `../../util/context.js` +- `../../services/BaseService.js` +- `../../config.js` +- `../../middleware/auth.js` +- `../../util/strutil.js` +- `../../fun/dev-console-ui-utils.js` +- `../../helpers.js` +- `../../fun/logos.js` diff --git a/src/backend/src/modules/web/SocketioService.js b/src/backend/src/modules/web/SocketioService.js new file mode 100644 index 0000000000..01098a7862 --- /dev/null +++ b/src/backend/src/modules/web/SocketioService.js @@ -0,0 +1,75 @@ +// METADATA // {"ai-params":{"service":"claude"},"ai-commented":{"service":"claude"}} +const BaseService = require('../../services/BaseService'); + +/** + * SocketioService provides a service for sending messages to clients. + * socket.io is used behind the scenes. This service provides a simpler + * interface for sending messages to rooms or socket ids. + */ +class SocketioService extends BaseService { + static MODULES = { + socketio: require('socket.io'), + }; + + /** + * Initializes socket.io + * + * @evtparam server The server to attach socket.io to. + */ + ['__on_install.socketio'] (_, { server }) { + const require = this.require; + + const socketio = require('socket.io'); + /** + * @type {import('socket.io').Server} + */ + this.io = socketio(server, { + cors: { + origin: '*', + } + }); + } + + + /** + * Sends a message to specified socket(s) or room(s) + * + * @param {Array|Object} socket_specifiers - Single or array of objects specifying target sockets/rooms + * @param {string} key - The event key/name to emit + * @param {*} data - The data payload to send + * @returns {Promise} + */ + async send (socket_specifiers, key, data) { + if ( ! Array.isArray(socket_specifiers) ) { + socket_specifiers = [socket_specifiers]; + } + + for ( const socket_specifier of socket_specifiers ) { + if ( socket_specifier.room ) { + this.io.to(socket_specifier.room).emit(key, data); + } else if ( socket_specifier.socket ) { + const io = this.io.sockets.sockets.get(socket_specifier.socket) + if ( ! io ) continue; + io.emit(key, data); + } + } + } + + /** + * Checks if the specified socket or room exists + * + * @param {Object} socket_specifier - The socket specifier object + * @returns {boolean} True if the socket exists, false otherwise + */ + has (socket_specifier) { + if ( socket_specifier.room ) { + const room = this.io.sockets.adapter.rooms.get(socket_specifier.room); + return (!!room) && room.size > 0; + } + if ( socket_specifier.socket ) { + return this.io.sockets.sockets.has(socket_specifier.socket); + } + } +} + +module.exports = SocketioService; diff --git a/src/backend/src/modules/web/WebModule.js b/src/backend/src/modules/web/WebModule.js new file mode 100644 index 0000000000..b0d9860ae9 --- /dev/null +++ b/src/backend/src/modules/web/WebModule.js @@ -0,0 +1,30 @@ +const { AdvancedBase } = require("@heyputer/putility"); + +/** + * This module initializes a pre-configured web server and socket.io server. + * The main service, WebServerService, emits 'install.routes' and provides + * the server instance to the callback. + */ +class WebModule extends AdvancedBase { + async install (context) { + // === LIBS === // + const useapi = context.get('useapi'); + useapi.def('web', require('./lib/__lib__.js'), { assign: true }); + + // === SERVICES === // + const services = context.get('services'); + + const SocketioService = require("./SocketioService"); + services.registerService('socketio', SocketioService); + + const WebServerService = require("./WebServerService"); + services.registerService('web-server', WebServerService); + + const APIErrorService = require("./APIErrorService"); + services.registerService('api-error', APIErrorService); + } +} + +module.exports = { + WebModule, +}; diff --git a/src/backend/src/services/WebServerService.js b/src/backend/src/modules/web/WebServerService.js similarity index 73% rename from src/backend/src/services/WebServerService.js rename to src/backend/src/modules/web/WebServerService.js index 980fcadb88..9012a863e8 100644 --- a/src/backend/src/services/WebServerService.js +++ b/src/backend/src/modules/web/WebServerService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"openai-completion","model":"gpt-4o-mini"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -17,18 +18,25 @@ * along with this program. If not, see . */ const express = require('express'); -const eggspress = require("../api/eggspress"); -const { Context, ContextExpressMiddleware } = require("../util/context"); -const BaseService = require("./BaseService"); +const eggspress = require("./lib/eggspress.js"); +const { Context, ContextExpressMiddleware } = require("../../util/context.js"); +const BaseService = require("../../services/BaseService.js"); -const config = require('../config'); +const config = require('../../config.js'); const https = require('https') var http = require('http'); const fs = require('fs'); -const auth = require('../middleware/auth'); -const { osclink } = require('../util/strutil'); -const { surrounding_box, es_import_promise } = require('../fun/dev-console-ui-utils'); +const auth = require('../../middleware/auth.js'); +const { surrounding_box, es_import_promise } = require('../../fun/dev-console-ui-utils.js'); +const relative_require = require; +const strutil = require('@heyputer/putility').libs.string; + +/** +* This class, WebServerService, is responsible for starting and managing the Puter web server. +* It initializes the Express app, sets up middlewares, routes, and handles authentication and web sockets. +* It also validates the host header and IP addresses to prevent security vulnerabilities. +*/ class WebServerService extends BaseService { static MODULES = { https: require('https'), @@ -42,6 +50,15 @@ class WebServerService extends BaseService { morgan: require('morgan'), }; + + /** + * This method initializes the backend web server for Puter. It sets up the Express app, configures middleware, and starts the HTTP server. + * + * @param {Express} app - The Express app instance to configure. + * @returns {void} + * @private + */ + // comment above line 44 in WebServerService.js async ['__on_boot.consolidation'] () { const app = this.app; const services = this.services; @@ -51,8 +68,18 @@ class WebServerService extends BaseService { router_webhooks: this.router_webhooks, }); await services.emit('install.routes-gui', { app }); + + this.log.noticeme('web server setup done'); } + + /** + * Starts the web server and listens for incoming connections. + * This method sets up the Express app, sets up middleware, and starts the server on the specified port. + * It also sets up the Socket.io server for real-time communication. + * + * @returns {Promise} A promise that resolves once the server is started. + */ async ['__on_boot.activation'] () { const services = this.services; await services.emit('start.webserver'); @@ -60,15 +87,24 @@ class WebServerService extends BaseService { this.print_puter_logo_(); } + + /** + * This method starts the web server by listening on the specified port. It tries multiple ports if the first one is in use. + * If the `config.http_port` is set to 'auto', it will try to find an available port in a range of 4100 to 4299. + * Once the server is up and running, it emits the 'start.webserver' and 'ready.webserver' events. + * If the `config.env` is set to 'dev' and `config.no_browser_launch` is false, it will open the Puter URL in the default browser. + * + * @return {Promise} A promise that resolves when the server is up and running. + */ async ['__on_start.webserver'] () { await es_import_promise; // error handling middleware goes last, as per the // expressjs documentation: // https://expressjs.com/en/guide/error-handling.html - this.app.use(require('../api/api_error_handler')); + this.app.use(require('./lib/api_error_handler.js')); - const { jwt_auth } = require('../helpers'); + const { jwt_auth } = require('../../helpers.js'); config.http_port = process.env.PORT ?? config.http_port; @@ -80,6 +116,14 @@ class WebServerService extends BaseService { let server; const auto_port = config.http_port === 'auto'; + /** + * Initializes the web server and starts listening for incoming requests. + * + * @param {Object} services - An object containing other services such as logger, config, etc. + */ + WebServerService.prototype._initWebServer = function (services) { + // Implementation goes here + }; let ports_to_try = auto_port ? (() => { const ports = []; for ( let i = 0 ; i < 20 ; i++ ) { @@ -108,6 +152,15 @@ class WebServerService extends BaseService { rjct(e); } }); + /** + * Starts the web server. + * + * This method is responsible for creating the HTTP server, setting up middleware, and starting the server on the specified port. If the specified port is "auto", it will attempt to find an available port within a range. + * + * @returns {Promise} + */ + // Add this comment above line 110 + // (line 110 of the provided code) server.on('listening', () => { rslv(); }) @@ -137,9 +190,17 @@ class WebServerService extends BaseService { console.log('Error opening browser', e); } } + /** + * Starts the HTTP server. + * + * This method sets up the Express server, initializes middleware, and starts the HTTP server. + * It handles error handling, authentication, and other necessary configurations. + * + * @returns {Promise} A Promise that resolves when the server is listening. + */ this.startup_widget = () => { - const link = `\x1B[34;1m${osclink(url)}\x1B[0m`; + const link = `\x1B[34;1m${strutil.osclink(url)}\x1B[0m`; const lines = [ `Puter is now live at: ${link}`, `Type web:dismiss to un-stick this message`, @@ -162,7 +223,11 @@ class WebServerService extends BaseService { // server.keepAliveTimeout = 1000 * 60 * 60 * 2; // 2 hours // Socket.io server instance - const socketio = require('../socketio.js').init(server); + // const socketio = require('../../socketio.js').init(server); + + // TODO: ^ Replace above line with the following code: + await this.services.emit('install.socketio', { server }); + const socketio = this.services.get('socketio').io; // Socket.io middleware for authentication socketio.use(async (socket, next) => { @@ -170,6 +235,7 @@ class WebServerService extends BaseService { try { let auth_res = await jwt_auth(socket); // successful auth + socket.actor = auth_res.actor; socket.user = auth_res.user; socket.token = auth_res.token; // join user room @@ -186,16 +252,32 @@ class WebServerService extends BaseService { } }); + const context = Context.get(); socketio.on('connection', (socket) => { + /** + * Starts the web server and associated services. + * + * This method is responsible for starting the web server and its associated services. It first initializes the middlewares and routes for the server, then begins the server with the specified HTTP port. If the specified port is not available, it will try to find an available port within a range. + * + * @returns {Promise} A promise that resolves when the server is started. + */ + // eslint-disable-next-line no-unused-vars + WebServerService.prototype.__on_start_webserver = async function () { + // ... + }; socket.on('disconnect', () => { }); socket.on('trash.is_empty', (msg) => { socket.broadcast.to(socket.user.id).emit('trash.is_empty', msg); }); - socket.on('puter_is_actually_open', (msg) => { + socket.on('puter_is_actually_open', async (msg) => { const svc_event = this.services.get('event'); - svc_event.emit('web.socket.user-connected', { - user: socket.user + await context.sub({ + actor: socket.actor, + }).arun(async () => { + await svc_event.emit('web.socket.user-connected', { + user: socket.user + }); }); }); }); @@ -204,10 +286,23 @@ class WebServerService extends BaseService { await this.services.emit('install.websockets'); } + + /** + * Starts the Puter web server and sets up routes, middleware, and error handling. + * + * @param {object} services - An object containing all services available to the web server. + * @returns {Promise} A promise that resolves when the web server is fully started. + */ get_server () { return this.server_; } + + /** + * Handles starting and managing the Puter web server. + * + * @param {Object} services - An object containing all services. + */ async _init () { const app = express(); this.app = app; @@ -219,13 +314,13 @@ class WebServerService extends BaseService { const require = this.require; - + const config = this.global_config; new ContextExpressMiddleware({ parent: globalThis.root_context.sub({ puter_environment: Context.create({ env: config.env, - version: require('../../package.json').version, + version: relative_require('../../../package.json').version, }), }, 'mw') }).install(app); @@ -274,6 +369,16 @@ class WebServerService extends BaseService { app.use(morgan(':method :url :status :response-time', { stream })); } + + /** + * Initialize the web server, start it, and handle any related logic. + * + * This method is responsible for creating the server and listening on the + * appropriate port. It also sets up middleware, routes, and other necessary + * configurations. + * + * @returns {Promise} A promise that resolves once the server is up and running. + */ app.use((() => { // const router = express.Router(); // router.get('/wut', express.json(), (req, res, next) => { @@ -292,6 +397,14 @@ class WebServerService extends BaseService { (() => { const onFinished = require('on-finished'); app.use((req, res, next) => { + /** + * Starts the web server and sets up routes, middleware, and web sockets. + * + * @returns {Promise} Resolves once the server is up and running. + */ + WebServerService.prototype._initWebServer = async function() { + // Your comment here + }; onFinished(res, () => { if ( res.statusCode !== 500 ) return; if ( req.__error_handled ) return; @@ -495,9 +608,19 @@ class WebServerService extends BaseService { ]); } + + /** + * Starts the web server and sets up the necessary middleware and routes. + * This method is responsible for initializing the Express app, handling authentication, + * setting up routes, and starting the HTTP server. It also sets up error handling and + * socket.io for real-time communication. + * + * @param {Object} services - The services object containing all necessary services. + */ + // comment above line 497 print_puter_logo_() { if ( this.global_config.env !== 'dev' ) return; - const logos = require('../fun/logos.js'); + const logos = require('../../fun/logos.js'); let last_logo = undefined; for ( const logo of logos ) { if ( logo.sz <= (process.stdout.columns ?? 0) ) { @@ -508,7 +631,6 @@ class WebServerService extends BaseService { const lines = last_logo.txt.split('\n'); const width = process.stdout.columns; const pad = (width - last_logo.sz) / 2; - const asymmetrical = pad % 1 !== 0; const pad_left = Math.floor(pad); const pad_right = Math.ceil(pad); for ( let i = 0 ; i < lines.length ; i++ ) { diff --git a/src/backend/src/modules/web/lib/__lib__.js b/src/backend/src/modules/web/lib/__lib__.js new file mode 100644 index 0000000000..2dc2a60ecd --- /dev/null +++ b/src/backend/src/modules/web/lib/__lib__.js @@ -0,0 +1,4 @@ +module.exports = { + eggspress: require("./eggspress"), + api_error_handler: require("./api_error_handler"), +}; diff --git a/src/backend/src/modules/web/lib/api_error_handler.js b/src/backend/src/modules/web/lib/api_error_handler.js new file mode 100644 index 0000000000..c711392c95 --- /dev/null +++ b/src/backend/src/modules/web/lib/api_error_handler.js @@ -0,0 +1,75 @@ +/* + * Copyright (C) 2024 Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +const APIError = require('../../../api/APIError.js'); + +/** + * api_error_handler() is an express error handler for API errors. + * It adheres to the express error handler signature and should be + * used as the last middleware in an express app. + * + * Since Express 5 is not yet released, this function is used by + * eggspress() to handle errors instead of as a middleware. + * + * @param {*} err + * @param {*} req + * @param {*} res + * @param {*} next + * @returns + */ +module.exports = function api_error_handler (err, req, res, next) { + if (res.headersSent) { + console.error('error after headers were sent:', err); + return next(err) + } + + // API errors might have a response to help the + // developer resolve the issue. + if ( err instanceof APIError ) { + return err.write(res); + } + + if ( + typeof err === 'object' && + ! (err instanceof Error) && + err.hasOwnProperty('message') + ) { + const apiError = APIError.create(400, err); + return apiError.write(res); + } + + console.error('internal server error:', err); + + const services = globalThis.services; + if ( services && services.has('alarm') ) { + const alarm = services.get('alarm'); + alarm.create('api_error_handler', err.message, { + error: err, + url: req.url, + method: req.method, + body: req.body, + headers: req.headers, + }); + } + + req.__error_handled = true; + + // Other errors should provide as little information + // to the client as possible for security reasons. + return res.send(500, 'Internal Server Error'); +}; diff --git a/src/backend/src/modules/web/lib/eggspress.js b/src/backend/src/modules/web/lib/eggspress.js new file mode 100644 index 0000000000..5cfb220fba --- /dev/null +++ b/src/backend/src/modules/web/lib/eggspress.js @@ -0,0 +1,199 @@ +/* + * Copyright (C) 2024 Puter Technologies Inc. + * + * This file is part of Puter. + * + * Puter is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as published + * by the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +const express = require('express'); +const multer = require('multer'); +const multest = require('@heyputer/multest'); +const api_error_handler = require('./api_error_handler.js'); + +const fsBeforeMW = require('../../../middleware/fs.js'); +const APIError = require('../../../api/APIError.js'); +const { Context } = require('../../../util/context.js'); +const { subdomain } = require('../../../helpers.js'); + +/** + * eggspress() is a factory function for creating express routers. + * + * @param {*} route the route to the router + * @param {*} settings the settings for the router. The following + * properties are supported: + * - auth: whether or not to use the auth middleware + * - fs: whether or not to use the fs middleware + * - json: whether or not to use the json middleware + * - customArgs: custom arguments to pass to the router + * - allowedMethods: the allowed HTTP methods + * @param {*} handler the handler for the router + * @returns {express.Router} the router + */ +module.exports = function eggspress (route, settings, handler) { + const router = express.Router(); + const mw = []; + const afterMW = []; + + // These flags enable specific middleware. + if ( settings.abuse ) mw.push(require('../../../middleware/abuse')(settings.abuse)); + if ( settings.auth ) mw.push(require('../../../middleware/auth')); + if ( settings.auth2 ) mw.push(require('../../../middleware/auth2')); + if ( settings.fs ) { + mw.push(fsBeforeMW); + } + if ( settings.verified ) mw.push(require('../../../middleware/verified')); + if ( settings.json ) mw.push(express.json()); + + // The `files` setting is an array of strings. Each string is the name + // of a multipart field that contains files. `multer` is used to parse + // the multipart request and store the files in `req.files`. + if ( settings.files ) { + for ( const key of settings.files ) { + mw.push(multer().array(key)); + } + } + + if ( settings.multest ) { + mw.push(multest()); + } + + // The `multipart_jsons` setting is an array of strings. Each string + // is the name of a multipart field that contains JSON. This middleware + // parses the JSON in each field and stores the result in `req.body`. + if ( settings.multipart_jsons ) { + for ( const key of settings.multipart_jsons ) { + mw.push((req, res, next) => { + try { + if ( ! Array.isArray(req.body[key]) ) { + req.body[key] = [JSON.parse(req.body[key])]; + } else { + req.body[key] = req.body[key].map(JSON.parse); + } + } catch (e) { + return res.status(400).send({ + error: { + message: `Invalid JSON in multipart field ${key}` + } + }); + } + next(); + }); + } + } + + // The `alias` setting is an object. Each key is the name of a + // parameter. Each value is the name of a parameter that should + // be aliased to the key. + if ( settings.alias ) { + for ( const alias in settings.alias ) { + const target = settings.alias[alias]; + mw.push((req, res, next) => { + const values = req.method === 'GET' ? req.query : req.body; + if ( values[alias] ) { + values[target] = values[alias]; + } + next(); + }); + } + } + + // The `parameters` setting is an object. Each key is the name of a + // parameter. Each value is a `Param` object. The `Param` object + // specifies how to validate the parameter. + if ( settings.parameters ) { + for ( const key in settings.parameters ) { + const param = settings.parameters[key]; + mw.push(async (req, res, next) => { + if ( ! req.values ) req.values = {}; + + const values = req.method === 'GET' ? req.query : req.body; + const getParam = (key) => values[key]; + try { + const result = await param.consolidate({ req, getParam }); + req.values[key] = result; + } catch (e) { + api_error_handler(e, req, res, next); + return; + } + next(); + }); + } + } + + // what if I wanted to pass arguments to, for example, `json`? + if ( settings.customArgs ) mw.push(settings.customArgs); + + if ( settings.alarm_timeout ) { + mw.push((req, res, next) => { + setTimeout(() => { + if ( ! res.headersSent ) { + const log = req.services.get('log-service').create('eggspress:timeout'); + const errors = req.services.get('error-service').create(log); + let id = Array.isArray(route) ? route[0] : route; + id = id.replace(/\//g, '_'); + errors.report(id, { + source: new Error('Response timed out.'), + message: 'Response timed out.', + trace: true, + alarm: true, + }); + } + }, settings.alarm_timeout); + next(); + }); + } + + if ( settings.response_timeout ) { + mw.push((req, res, next) => { + setTimeout(() => { + if ( ! res.headersSent ) { + api_error_handler(APIError.create('response_timeout'), req, res, next); + } + }, settings.response_timeout); + next(); + }); + } + + if ( settings.mw ) mw.push(...settings.mw); + + const errorHandledHandler = async function (req, res, next) { + if ( settings.subdomain ) { + if ( subdomain(req) !== settings.subdomain ) { + return next(); + } + } + try { + const expected_ctx = res.locals.ctx; + const received_ctx = Context.get(undefined, { allow_fallback: true }); + + if ( expected_ctx != received_ctx ) { + await expected_ctx.arun(async () => { + await handler(req, res, next); + }); + } else await handler(req, res, next); + } catch (e) { + api_error_handler(e, req, res, next); + } + }; + + if ( settings.allowedMethods.includes('GET') ) { + router.get(route, ...mw, errorHandledHandler, ...afterMW); + } + + if ( settings.allowedMethods.includes('POST') ) { + router.post(route, ...mw, errorHandledHandler, ...afterMW); + } + + return router; +} \ No newline at end of file diff --git a/src/backend/src/om/entitystorage/AppES.js b/src/backend/src/om/entitystorage/AppES.js index 91f821f7ff..1aab485978 100644 --- a/src/backend/src/om/entitystorage/AppES.js +++ b/src/backend/src/om/entitystorage/AppES.js @@ -129,6 +129,23 @@ class AppES extends BaseES { await this.db.write(stmt, rows.flat()); } + const has_new_icon = + ( ! extra.old_entity ) || ( + await entity.get('icon') !== await extra.old_entity.get('icon') + ); + + if ( has_new_icon ) { + const svc_event = this.context.get('services').get('event'); + const event = { + app_uid: await entity.get('uid'), + data_url: await entity.get('icon'), + }; + await svc_event.emit('app.new-icon', event); + if ( event.url ) { + await entity.set('icon') + } + } + // Associate app with subdomain (if applicable) if ( subdomain_id ) { await this.db.write( diff --git a/src/backend/src/routers/_default.js b/src/backend/src/routers/_default.js index 2ec82d5eb0..5a6071ebc5 100644 --- a/src/backend/src/routers/_default.js +++ b/src/backend/src/routers/_default.js @@ -22,7 +22,6 @@ const config = require('../config'); const router = express.Router(); const _path = require('path'); const _fs = require('fs'); -const auth = require('../middleware/auth.js'); const { Context } = require('../util/context'); const { DB_READ } = require('../services/database/consts'); const { PathBuilder } = require('../util/pathutil.js'); @@ -289,10 +288,8 @@ router.all('*', async function(req, res, next) { invalidate_cached_user(user); // send realtime success msg to client - let socketio = require('../socketio.js').getio(); - if(socketio){ - socketio.to(user.id).emit('user.email_confirmed', {}) - } + const svc_socketio = req.services.get('socketio'); + svc_socketio.send({ room: user.id }, 'user.email_confirmed', {}); // return results h += `

Your email has been successfully confirmed.

`; diff --git a/src/backend/src/routers/auth/configure-2fa.js b/src/backend/src/routers/auth/configure-2fa.js index 9c3c461e4d..9ee9af08db 100644 --- a/src/backend/src/routers/auth/configure-2fa.js +++ b/src/backend/src/routers/auth/configure-2fa.js @@ -37,8 +37,6 @@ module.exports = eggspress('/auth/configure-2fa/:action', { throw APIError.create('forbidden'); } - const user = actor.type.user; - const actions = {}; const db = await x.get('services').get('database').get(DB_WRITE, '2fa'); diff --git a/src/backend/src/routers/auth/grant-user-app.js b/src/backend/src/routers/auth/grant-user-app.js index ace7c67bf3..4d817d059c 100644 --- a/src/backend/src/routers/auth/grant-user-app.js +++ b/src/backend/src/routers/auth/grant-user-app.js @@ -50,7 +50,7 @@ module.exports = eggspress('/auth/grant-user-app', { }); } - const token = await svc_permission.grant_user_app_permission( + await svc_permission.grant_user_app_permission( actor, req.body.app_uid, req.body.permission, req.body.extra || {}, req.body.meta || {} ); diff --git a/src/backend/src/routers/auth/revoke-user-app.js b/src/backend/src/routers/auth/revoke-user-app.js index 853f0b2b42..fe5bb46ab9 100644 --- a/src/backend/src/routers/auth/revoke-user-app.js +++ b/src/backend/src/routers/auth/revoke-user-app.js @@ -50,7 +50,7 @@ module.exports = eggspress('/auth/revoke-user-app', { ); } - const token = await svc_permission.revoke_user_app_permission( + await svc_permission.revoke_user_app_permission( actor, req.body.app_uid, req.body.permission, req.body.meta || {}, ); diff --git a/src/backend/src/routers/change_email.js b/src/backend/src/routers/change_email.js index 2c0c190222..52870eba09 100644 --- a/src/backend/src/routers/change_email.js +++ b/src/backend/src/routers/change_email.js @@ -75,12 +75,16 @@ const CHANGE_EMAIL_CONFIRM = eggspress('/change_email/confirm', { 'UPDATE `user` SET `email` = ?, `clean_email` = ?, `unconfirmed_change_email` = NULL, `change_email_confirm_token` = NULL, `pass_recovery_token` = NULL WHERE `id` = ?', [new_email, clean_email, user_id] ); + + const svc_event = req.services.get('event'); + svc_event.emit('user.email-changed', { + user_id: user_id, + new_email, + }); invalidate_cached_user_by_id(user_id); - let socketio = require('../socketio.js').getio(); - if(socketio){ - socketio.to(user_id).emit('user.email_changed', {}) - } + const svc_socketio = req.services.get('socketio'); + svc_socketio.send({ room: user_id }, 'user.email_changed', {}); const h = `

Your email has been successfully confirmed.

`; return res.send(h); diff --git a/src/backend/src/routers/confirm-email.js b/src/backend/src/routers/confirm-email.js index d8269e2b0a..922e71d6f3 100644 --- a/src/backend/src/routers/confirm-email.js +++ b/src/backend/src/routers/confirm-email.js @@ -87,14 +87,14 @@ router.post('/confirm-email', auth, express.json(), async (req, res, next)=>{ // Send realtime success msg to client if(req.body.code === req.user.email_confirm_code){ - let socketio = require('../socketio.js').getio(); - if(socketio){ - socketio.to(req.user.id).emit('user.email_confirmed', {original_client_socket_id: req.body.original_client_socket_id}) - } + const svc_socketio = req.services.get('socketio'); + svc_socketio.send({ room: req.user.id }, 'user.email_confirmed', { + original_client_socket_id: req.body.original_client_socket_id + }); } // return results return res.send(res_obj) }) -module.exports = router \ No newline at end of file +module.exports = router diff --git a/src/backend/src/routers/drivers/call.js b/src/backend/src/routers/drivers/call.js index 9e472d7a25..f0453b9958 100644 --- a/src/backend/src/routers/drivers/call.js +++ b/src/backend/src/routers/drivers/call.js @@ -23,7 +23,7 @@ const { TypeSpec } = require("../../services/drivers/meta/Construct"); const { TypedValue } = require("../../services/drivers/meta/Runtime"); const { Context } = require("../../util/context"); const { whatis } = require("../../util/langutil"); -const { TeePromise } = require("../../util/promise"); +const { TeePromise } = require('@heyputer/putility').libs.promise; const { valid_file_size } = require("../../util/validutil"); let _handle_multipart; diff --git a/src/backend/src/routers/filesystem_api/batch/PathResolver.js b/src/backend/src/routers/filesystem_api/batch/PathResolver.js index d2b75d1d4e..e4d3d9c1ca 100644 --- a/src/backend/src/routers/filesystem_api/batch/PathResolver.js +++ b/src/backend/src/routers/filesystem_api/batch/PathResolver.js @@ -17,7 +17,7 @@ * along with this program. If not, see . */ const APIError = require('../../../api/APIError.js'); -const { relativeSelector, NodeUIDSelector } = require('../../../filesystem/node/selectors.js'); +const { relativeSelector } = require('../../../filesystem/node/selectors.js'); const ERR_INVALID_PATHREF = 'Invalid path reference in path: '; const ERR_UNKNOWN_PATHREF = 'Unknown path reference in path: '; diff --git a/src/backend/src/routers/filesystem_api/batch/all.js b/src/backend/src/routers/filesystem_api/batch/all.js index f92d1f77d6..700bea6630 100644 --- a/src/backend/src/routers/filesystem_api/batch/all.js +++ b/src/backend/src/routers/filesystem_api/batch/all.js @@ -19,19 +19,15 @@ const APIError = require("../../../api/APIError"); const eggspress = require("../../../api/eggspress"); const config = require("../../../config"); -const PathResolver = require("./PathResolver"); -const { WorkUnit } = require("../../../services/runtime-analysis/ExpectationService"); const { Context } = require("../../../util/context"); const Busboy = require('busboy'); const { BatchExecutor } = require("../../../filesystem/batch/BatchExecutor"); -const { TeePromise } = require("../../../util/promise"); -const { EWMA, MovingMode } = require("../../../util/opmath"); +const { TeePromise } = require('@heyputer/putility').libs.promise; +const { MovingMode } = require("../../../util/opmath"); const { get_app } = require('../../../helpers'); const { valid_file_size } = require("../../../util/validutil"); const { OnlyOnceFn } = require("../../../util/fnutil.js"); -const commands = require('../../../filesystem/batch/commands.js').commands; - module.exports = eggspress('/batch', { subdomain: 'api', verified: true, @@ -138,8 +134,6 @@ module.exports = eggspress('/batch', { const pending_operations = []; const response_promises = []; const fileinfos = []; - let total = 0; - let total_tbd = true; const on_nonfile_data_end = OnlyOnceFn(() => { if ( request_error ) { @@ -224,10 +218,6 @@ module.exports = eggspress('/batch', { } }); - let i = 0; - let ended = []; - let ps = []; - busboy.on('file', async (fieldname, stream, detais) => { if ( batch_exe.total_tbd ) { batch_exe.total_tbd = false; diff --git a/src/backend/src/routers/filesystem_api/batch/prepare.js b/src/backend/src/routers/filesystem_api/batch/prepare.js deleted file mode 100644 index f3cb595011..0000000000 --- a/src/backend/src/routers/filesystem_api/batch/prepare.js +++ /dev/null @@ -1,18 +0,0 @@ -/* - * Copyright (C) 2024 Puter Technologies Inc. - * - * This file is part of Puter. - * - * Puter is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as published - * by the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - */ diff --git a/src/backend/src/routers/filesystem_api/copy.js b/src/backend/src/routers/filesystem_api/copy.js index 9860c0f1d4..e8b0c19291 100644 --- a/src/backend/src/routers/filesystem_api/copy.js +++ b/src/backend/src/routers/filesystem_api/copy.js @@ -19,12 +19,8 @@ "use strict" const eggspress = require('../../api/eggspress.js'); const FSNodeParam = require('../../api/filesystem/FSNodeParam.js'); -const _path = require('path'); -const { NodeUIDSelector } = require('../../filesystem/node/selectors.js'); const { HLCopy } = require('../../filesystem/hl_operations/hl_copy.js'); const { Context } = require('../../util/context.js'); -const { DatabaseFSEntryService } = require('../../filesystem/storage/DatabaseFSEntryService.js'); -const { ProxyContainer } = require('../../services/Container.js'); // -----------------------------------------------------------------------// // POST /copy @@ -40,16 +36,12 @@ module.exports = eggspress('/copy', { source: new FSNodeParam('source'), destination: new FSNodeParam('destination'), } -}, async (req, res, next) => { +}, async (req, res) => { const user = req.user const dedupe_name = req.body.dedupe_name ?? req.body.change_name ?? false; - // // check if source would be an ancestor of destination - // if((abs_dest_path + '/').startsWith(abs_source_path + '/')){ - // return res.status(400).send('Can not copy a item into itself.') - // } let frame; { const x = Context.get(); @@ -66,23 +58,8 @@ module.exports = eggspress('/copy', { x.set(operationTraceSvc.ckey('frame'), frame); } - const log = req.services.get('log-service').create('copy'); - const filesystem = req.services.get('filesystem'); - - // copy - const {get_app, uuid2fsentry, is_shared_with_anyone, suggest_app_for_fsentry} = require('../../helpers.js') - let new_fsentries = []; - const tracer = req.services.get('traceService').tracer; await tracer.startActiveSpan('filesystem_api.copy', async span => { - // const op = await filesystem.cp(req.fs, { - // source: req.values.source, - // destinationOrParent: req.values.destination, - // user: user, - // new_name: req.body.new_name, - // overwrite: req.body.overwrite ?? false, - // dedupe_name, - // }); // === upcoming copy behaviour === const hl_copy = new HLCopy(); diff --git a/src/backend/src/routers/filesystem_api/delete.js b/src/backend/src/routers/filesystem_api/delete.js index 076ff79a35..b77cbae711 100644 --- a/src/backend/src/routers/filesystem_api/delete.js +++ b/src/backend/src/routers/filesystem_api/delete.js @@ -48,15 +48,12 @@ module.exports = eggspress('/delete', { else if(paths.length === 0) return res.status(400).send('paths cannot be empty') - const socketio = require('../../socketio.js').getio(); - // try to delete each path in the array one by one (if glob, resolve first) // TODO: remove this pseudo-batch - for(let j=0; j < paths.length; j++){ - let item_path = paths[j]; + for ( const item_path of paths ) { const target = await (new FSNodeParam('path')).consolidate({ req: { fs: req.fs, user }, - getParam: () => paths[j], + getParam: () => item_path, }); const hl_remove = new HLRemove(); await hl_remove.run({ @@ -67,9 +64,11 @@ module.exports = eggspress('/delete', { }); // send realtime success msg to client - if(socketio){ - socketio.to(req.user.id).emit('item.removed', {path: item_path, descendants_only: descendants_only}) - } + const svc_socketio = req.services.get('socketio'); + svc_socketio.send({ room: req.user.id }, 'item.removed', { + path: item_path, + descendants_only: descendants_only, + }); } res.send({}); diff --git a/src/backend/src/routers/filesystem_api/readdir.js b/src/backend/src/routers/filesystem_api/readdir.js index 180b09ceb6..0476cca116 100644 --- a/src/backend/src/routers/filesystem_api/readdir.js +++ b/src/backend/src/routers/filesystem_api/readdir.js @@ -17,16 +17,10 @@ * along with this program. If not, see . */ "use strict" -const express = require('express'); -const router = express.Router(); -const auth = require('../../middleware/auth.js'); -const config = require('../../config.js'); -const PerformanceMonitor = require('../../monitor/PerformanceMonitor.js'); const { Context } = require('../../util/context.js'); const eggspress = require('../../api/eggspress.js'); const FSNodeParam = require('../../api/filesystem/FSNodeParam.js'); const FlagParam = require('../../api/filesystem/FlagParam.js'); -const { LLReadDir } = require('../../filesystem/ll_operations/ll_readdir.js'); const { HLReadDir } = require('../../filesystem/hl_operations/hl_readdir.js'); // -----------------------------------------------------------------------// @@ -47,35 +41,12 @@ module.exports = eggspress('/readdir', { no_assocs: new FlagParam('no_assocs', { optional: true }), } }, async (req, res, next) => { - const monitor = PerformanceMonitor.createContext("router.readdir"); - let log; { const x = Context.get(); log = x.get('services').get('log-service').create('readdir'); log.info(`readdir: ${req.body.path}`); } - // // `path` validation - // if(req.body.path === undefined) - // return res.status(400).send('path is required.') - // else if(req.body.path === '') - // return res.status(400).send('path cannot be empty.') - // else if(req.body.path === null) - // return res.status(400).send('path cannot be null.') - // else if(typeof req.body.path !== 'string') - // return res.status(400).send('path must be a string.') - - // if ( req.body.path.startsWith('~') ) { - // const homedir = `/${req.user.username}`; - // req.body.path = homedir + req.body.path.slice(1); - // } - - // `recursive` validation - // if(req.body.recursive !== undefined && typeof req.body.recursive !== 'boolean') - // return res.status(400).send('recursive must be a boolean.') - // else if(req.body.recursive === undefined) - // req.body.recursive = false; // default value - const subject = req.values.subject; const recursive = req.values.recursive; const no_thumbs = req.values.no_thumbs; diff --git a/src/backend/src/routers/filesystem_api/rename.js b/src/backend/src/routers/filesystem_api/rename.js index b2632178b0..9a44d14dd2 100644 --- a/src/backend/src/routers/filesystem_api/rename.js +++ b/src/backend/src/routers/filesystem_api/rename.js @@ -56,7 +56,7 @@ module.exports = eggspress('/rename', { // modules const db = req.services.get('database').get(DB_WRITE, 'filesystem'); const mime = require('mime-types'); - const {get_app, validate_fsentry_name, uuid2fsentry, chkperm, id2path} = require('../../helpers.js'); + const {get_app, validate_fsentry_name, id2path} = require('../../helpers.js'); const _path = require('path'); // new_name validation @@ -168,16 +168,14 @@ module.exports = eggspress('/rename', { is_dir: fsentry.is_dir, path: new_path, old_path: old_path, - type: contentType ? contentType : null, + type: contentType || null, associated_app: associated_app, original_client_socket_id: req.body.original_client_socket_id, }; // send realtime success msg to client - let socketio = require('../../socketio.js').getio(); - if(socketio){ - socketio.to(req.user.id).emit('item.renamed', return_obj) - } + const svc_socketio = req.services.get('socketio'); + svc_socketio.send({ room: req.user.id }, 'item.renamed', return_obj); return res.send(return_obj); }); diff --git a/src/backend/src/routers/filesystem_api/search.js b/src/backend/src/routers/filesystem_api/search.js index 379823b1e2..7241de957a 100644 --- a/src/backend/src/routers/filesystem_api/search.js +++ b/src/backend/src/routers/filesystem_api/search.js @@ -1,7 +1,5 @@ const eggspress = require("../../api/eggspress"); const { HLNameSearch } = require("../../filesystem/hl_operations/hl_name_search"); -const { subdomain } = require("../../helpers"); -const verified = require("../../middleware/verified"); module.exports = eggspress('/search', { subdomain: 'api', diff --git a/src/backend/src/routers/filesystem_api/write.js b/src/backend/src/routers/filesystem_api/write.js index 2bb289b7fc..ba06416b98 100644 --- a/src/backend/src/routers/filesystem_api/write.js +++ b/src/backend/src/routers/filesystem_api/write.js @@ -23,9 +23,8 @@ const { HLWrite } = require('../../filesystem/hl_operations/hl_write.js'); const { boolify } = require('../../util/hl_types.js'); const { Context } = require('../../util/context.js'); const Busboy = require('busboy'); -const { TeePromise } = require('../../util/promise.js'); +const { TeePromise } = require('@heyputer/putility').libs.promise; const APIError = require('../../api/APIError.js'); -const api_error_handler = require('../../api/api_error_handler.js'); const { valid_file_size } = require('../../util/validutil.js'); // -----------------------------------------------------------------------// @@ -54,10 +53,7 @@ module.exports = eggspress(['/up', '/write'], { }; // modules - const {get_app, mkdir} = require('../../helpers.js') - - // if(!req.files) - // return res.status(400).send('No files uploaded'); + const {get_app} = require('../../helpers.js') // Is this an entry for an app? let app; @@ -90,13 +86,6 @@ module.exports = eggspress(['/up', '/write'], { x.set(svc_clientOperation.ckey('tracker'), tracker); } - //------------------------------------------------------------- - // Variables used by busboy callbacks - //------------------------------------------------------------- - const on_first_file = () => { - frame_meta_ready(); - }; - //------------------------------------------------------------- // Multipart processing (using busboy) //------------------------------------------------------------- @@ -172,13 +161,8 @@ module.exports = eggspress(['/up', '/write'], { const values = req.method === 'GET' ? req.query : req.body; const getParam = (key) => values[key]; - try { - const result = await param.consolidate({ req, getParam }); - req.values[key] = result; - } catch (e) { - api_error_handler(e, req, res, next); - return; - } + const result = await param.consolidate({ req, getParam }); + req.values[key] = result; } if ( req.body.size === undefined ) { @@ -211,37 +195,4 @@ module.exports = eggspress(['/up', '/write'], { if ( frame ) frame.done(); return res.send(response); - - // upload files one by one - // for (let index = 0; index < req.files.length; index++) { - // let uploaded_file = req.files[index]; - - // // TEMP: create stream from buffer - // if ( uploaded_file.buffer ) { - // uploaded_file = { ...uploaded_file }; - // const buffer = uploaded_file.buffer; - // uploaded_file.stream = (() => { - // const { Readable } = require('stream'); - // return Readable.from(buffer); - // })(); - // delete uploaded_file.buffer; - // } - - // const hl_write = new HLWrite(); - // const response = await hl_write.run({ - // destination_or_parent: req.values.fsNode, - // specified_name: req.body.name, - // fallback_name: uploaded_file.originalname, - // overwrite: await boolify(req.body.overwrite), - // dedupe_name: await boolify(req.body.dedupe_name), - // shortcut_to: req.values.target, - - // create_missing_parents: boolify(req.body.create_missing_ancestors), - - // user: req.user, - // file: uploaded_file, - // }); - - // return res.send(response); - // } }); diff --git a/src/backend/src/routers/get-dev-profile.js b/src/backend/src/routers/get-dev-profile.js index 3b297d0df4..f97c556050 100644 --- a/src/backend/src/routers/get-dev-profile.js +++ b/src/backend/src/routers/get-dev-profile.js @@ -33,15 +33,27 @@ router.get('/get-dev-profile', auth, express.json(), async (req, response, next) // check if user is verified if((config.strict_email_verification_required || req.user.requires_email_confirmation) && !req.user.email_confirmed) return response.status(400).send({code: 'account_is_not_verified', message: 'Account is not verified'}); - + + // TODO: we currently invalidate the cache on every request, this is because a developer may + // have been approved for the incentive program from one server, but the cache on another server + // may not have been updated yet. This is a temporary solution until we implement a better way to + // handle this. The better way would be for different servers to communicate with each other + // when a developer is approved for the incentive program (or any other change that affects the + // cache) and update the cache on all servers. + require('../helpers').invalidate_cached_user(req.user); + const { get_user } = require('../helpers'); + + let dev = await get_user(req.user); + dev = dev ?? {}; + try{ // auth response.send({ - first_name: req.user.dev_first_name, - last_name: req.user.dev_last_name, - approved_for_incentive_program: req.user.dev_approved_for_incentive_program, - joined_incentive_program: req.user.dev_joined_incentive_program, - paypal: req.user.dev_paypal, + first_name: dev.dev_first_name, + last_name: dev.dev_last_name, + approved_for_incentive_program: dev.dev_approved_for_incentive_program, + joined_incentive_program: dev.dev_joined_incentive_program, + paypal: dev.dev_paypal, }); }catch(e){ console.log(e) diff --git a/src/backend/src/routers/get-launch-apps.js b/src/backend/src/routers/get-launch-apps.js index 754f7bd1cc..95d4ce654d 100644 --- a/src/backend/src/routers/get-launch-apps.js +++ b/src/backend/src/routers/get-launch-apps.js @@ -20,7 +20,6 @@ const express = require('express'); const router = express.Router(); const auth = require('../middleware/auth.js'); -const config = require('../config'); const { get_app } = require('../helpers.js'); const { DB_READ } = require('../services/database/consts.js'); @@ -28,8 +27,8 @@ const { DB_READ } = require('../services/database/consts.js'); // GET /get-launch-apps // -----------------------------------------------------------------------// router.get('/get-launch-apps', auth, express.json(), async (req, res, next)=>{ - let final_returned_obj = {}; - let retobj = []; + let result = {}; + // -----------------------------------------------------------------------// // Recent apps // -----------------------------------------------------------------------// @@ -46,37 +45,35 @@ router.get('/get-launch-apps', auth, express.json(), async (req, res, next)=>{ 'SELECT DISTINCT app_uid FROM app_opens WHERE user_id = ? GROUP BY app_uid ORDER BY MAX(_id) DESC LIMIT 10', [req.user.id]); // Update cache with the results from the db (if any results were returned) - if(apps && Array.isArray(apps) && apps.length > 0) + if(apps && Array.isArray(apps) && apps.length > 0) { kv.set('app_opens:user:' + req.user.id, apps); + } } - for (let index = 0; index < apps.length; index++) { - const app = await get_app({uid: apps[index].app_uid}); - let final_obj = {}; + // prepare each app for returning to user by only returning the necessary fields + // and adding them to the retobj array + result.recent = []; + console.log('\x1B[36;1m -------- RECENT APPS -------- \x1B[0m', apps); + for ( const { app_uid: uid } of apps ) { + console.log('\x1B[36;1m -------- UID -------- \x1B[0m', uid); + const app = await get_app({ uid }); + if ( ! app ) continue - // prepare each app for returning to user by only returning the necessary fields - // and adding them to the retobj array - if(app){ - final_obj = { - uuid: app.uid, - name: app.name, - title: app.title, - icon: app.icon, - godmode: app.godmode, - maximize_on_start: app.maximize_on_start, - index_url: app.index_url, - }; - } - // add to object to be returned - retobj.push(final_obj) + result.recent.push({ + uuid: app.uid, + name: app.name, + title: app.title, + icon: app.icon, + godmode: app.godmode, + maximize_on_start: app.maximize_on_start, + index_url: app.index_url, + }); } - final_returned_obj.recent = retobj; + // -----------------------------------------------------------------------// // Recommended apps // -----------------------------------------------------------------------// - // reset retobj - retobj = []; - let app_names = [ + let app_names = new Set([ 'app-center', 'dev-center', 'editor', @@ -100,49 +97,27 @@ router.get('/get-launch-apps', auth, express.json(), async (req, res, next)=>{ 'plushie-connect', 'hex-frvr', 'spider-solitaire', - ] + ]); // Prepare each app for returning to user by only returning the necessary fields // and adding them to the retobj array - if(app_names.length > 0){ - for (let index = 0; index < app_names.length; index++) { - const app = await get_app({name: app_names[index]}); - - let final_obj = {}; - if(app){ - final_obj = { - uuid: app.uid, - name: app.name, - title: app.title, - icon: app.icon, - godmode: app.godmode, - maximize_on_start: app.maximize_on_start, - index_url: app.index_url, - }; - } - // add to object to be returned - retobj.push(final_obj) - } - - // remove duplicates from retobj - if(retobj.length > 0) - retobj = retobj.filter((obj, pos, arr) => { - return arr.map(mapObj => mapObj['name']).indexOf(obj['name']) === pos; - }) - } + result.recommended = []; + for ( const name of app_names ) { + const app = await get_app({ name }); + if ( ! app ) continue; - // Order output based on input! - let final_obj = []; - for (let index = 0; index < app_names.length; index++) { - const app_name = app_names[index]; - for (let index = 0; index < retobj.length; index++) { - if(retobj[index].name === app_name) - final_obj.push(retobj[index]); - } + result.recommended.push({ + uuid: app.uid, + name: app.name, + title: app.title, + icon: app.icon, + godmode: app.godmode, + maximize_on_start: app.maximize_on_start, + index_url: app.index_url, + }); } - final_returned_obj.recommended = final_obj; - - return res.send(final_returned_obj); + return res.send(result); }) -module.exports = router \ No newline at end of file + +module.exports = router diff --git a/src/backend/src/routers/hosting/puter-site.js b/src/backend/src/routers/hosting/puter-site.js index a2680a6be2..d9d81bf48f 100644 --- a/src/backend/src/routers/hosting/puter-site.js +++ b/src/backend/src/routers/hosting/puter-site.js @@ -17,7 +17,7 @@ * along with this program. If not, see . */ const { AdvancedBase } = require("@heyputer/putility"); -const api_error_handler = require("../../api/api_error_handler"); +const api_error_handler = require("../../modules/web/lib/api_error_handler"); const config = require("../../config"); const { get_user, get_app, id2path } = require("../../helpers"); const { Context } = require("../../util/context"); diff --git a/src/backend/src/routers/rao.js b/src/backend/src/routers/rao.js index 0dfb8f598e..68b1aaca16 100644 --- a/src/backend/src/routers/rao.js +++ b/src/backend/src/routers/rao.js @@ -90,8 +90,8 @@ router.post('/rao', auth, express.json(), async (req, res, next)=>{ } // Update clients - const socketio = require('../socketio.js').getio(); - socketio.to(req.user.id).emit('app.opened', { + const svc_socketio = req.services.get('socketio'); + svc_socketio.send({ room: req.user.id }, 'app.opened', { uuid: opened_app.uid, uid: opened_app.uid, name: opened_app.name, diff --git a/src/backend/src/routers/signup.js b/src/backend/src/routers/signup.js index 75ecad0560..a07ba7b28b 100644 --- a/src/backend/src/routers/signup.js +++ b/src/backend/src/routers/signup.js @@ -17,14 +17,20 @@ * along with this program. If not, see . */ "use strict" -const express = require('express'); -const router = new express.Router(); -const {get_taskbar_items, generate_random_username, generate_system_fsentries, send_email_verification_code, send_email_verification_token, username_exists, invalidate_cached_user_by_id, get_user } = require('../helpers'); +const {get_taskbar_items, send_email_verification_code, send_email_verification_token, username_exists, invalidate_cached_user_by_id, get_user } = require('../helpers'); const config = require('../config'); const eggspress = require('../api/eggspress'); const { Context } = require('../util/context'); const { DB_WRITE } = require('../services/database/consts'); -const { can } = require('../util/langutil'); +const { generate_identifier } = require('../util/identifier'); + +async function generate_random_username () { + let username; + do { + username = generate_identifier(); + } while (await username_exists(username)); + return username; +} // -----------------------------------------------------------------------// // POST /signup @@ -239,15 +245,15 @@ module.exports = eggspress(['/signup'], { // audit_metadata JSON.stringify(audit_metadata), // signup_ip - req.connection.remoteAddress, + req.connection.remoteAddress ?? null, // signup_ip_fwd - req.headers['x-forwarded-for'], + req.headers['x-forwarded-for'] ?? null, // signup_user_agent - req.headers['user-agent'], + req.headers['user-agent'] ?? null, // signup_origin - req.headers['origin'], + req.headers['origin'] ?? null, // signup_server - config.server_id, + config.server_id ?? null, ] ); @@ -348,7 +354,8 @@ module.exports = eggspress(['/signup'], { } } - await generate_system_fsentries(user); + const svc_user = Context.get('services').get('user'); + await svc_user.generate_default_fsentries({ user }); //set cookie res.cookie(config.cookie_name, token, { diff --git a/src/backend/src/routers/writeFile.js b/src/backend/src/routers/writeFile.js index 7e42ec6fbf..44c4c41407 100644 --- a/src/backend/src/routers/writeFile.js +++ b/src/backend/src/routers/writeFile.js @@ -17,10 +17,7 @@ * along with this program. If not, see . */ "use strict" -const express = require('express'); -const router = new express.Router(); -const {uuid2fsentry, validate_signature_auth, get_url_from_req, sign_file} = require('../helpers'); -const fs = require('../middleware/fs.js'); +const {uuid2fsentry, validate_signature_auth, get_url_from_req, sign_file, get_user} = require('../helpers'); const { NodePathSelector, NodeUIDSelector } = require('../filesystem/node/selectors'); const eggspress = require('../api/eggspress'); const { HLWrite } = require('../filesystem/hl_operations/hl_write'); @@ -59,11 +56,6 @@ module.exports = eggspress('/writeFile', { return res.status(403).send(e); } - log.info('writeFile context: ' + ( - Context.get(undefined, { allow_fallback: true }) - ).describe()) - log.info('writeFile req context: ' + res.locals.ctx?.describe?.()); - // Get fsentry // todo this is done again in the following section, super inefficient let requested_item = await uuid2fsentry(req.query.uid); @@ -93,491 +85,57 @@ module.exports = eggspress('/writeFile', { if(owner_user.suspended) return res.status(401).send({error: 'Account suspended'}); - const db = req.services.get('database').get(DB_WRITE, 'filesystem'); - - // -----------------------------------------------------------------------// - // move - // -----------------------------------------------------------------------// - if(req.query.operation && req.query.operation === 'move'){ - console.log(req.body) - const { get_user } = require('../helpers') - const _path = require('path'); - const mime = require('mime-types'); - - // check if destination_write_url provided - if(!req.body.destination_write_url){ - return res.status(400).send({ - error:{ - message: 'No destination specified.' - } - }) - } - - // check if destination_write_url is valid - try{ - validate_signature_auth(req.body.destination_write_url, 'write'); - }catch(e){ - return res.status(403).send(e); - } - - try{ - const hl_move = new HLMove(); - - // TODO: [fs:operation:param-coercion] - const source_node = await (new FSNodeParam('uid')).consolidate({ - req, getParam: () => req.query.uid - }); - - // TODO: [fs:operation:param-coercion] - const dest_node = await (new FSNodeParam('dest_path')).consolidate({ - req, getParam: () => req.body.dest_path ?? req.body.destination_uid - }); - - const user = await get_user({id: await source_node.get('user_id')}); - - const opts = { - // TODO: [fs:normalize-writeFile-user] - user, - source: source_node, - destination_or_parent: dest_node, - overwrite: req.body.overwrite ?? false, - new_name: req.body.new_name, - new_metadata: req.body.new_metadata, - create_missing_parents: req.body.create_missing_parents, - }; - - // TODO: [fs:DRY-writeFile-context] - const r = await Context.get().sub({ actor: Actor.adapt(user) }).arun(async () => { - return await hl_move.run({ - ...opts, - actor: Context.get('actor'), + const writeFile_handler_api = { + async get_dest_node () { + if(!req.body.destination_write_url){ + res.status(400).send({ + error:{ + message: 'No destination specified.' + } }); - }); - - return res.send({ - ...r.moved, - old_path: r.old_path, - new_path: r.moved.path, - }); - }catch(e){ - console.log(e) - return res.status(400).send(e) - } - } - - // -----------------------------------------------------------------------// - // copy - // -----------------------------------------------------------------------// - else if(req.query.operation && req.query.operation === 'copy'){ - const {is_shared_with_anyone, suggest_app_for_fsentry, cp, validate_fsentry_name, convert_path_to_fsentry, uuid2fsentry, get_user, id2path, id2uuid} = require('../helpers') - const _path = require('path'); - const mime = require('mime-types'); - - // check if destination_write_url provided - if(!req.body.destination_write_url){ - return res.status(400).send({ - error:{ - message: 'No destination specified.' - } - }) - } - - // check if destination_write_url is valid - try{ - validate_signature_auth(req.body.destination_write_url, 'write'); - }catch(e){ - return res.status(403).send(e); - } - - const overwrite = req.body.overwrite ?? false; - const change_name = req.body.auto_rename ?? false; - - // TODO: [fs:operation:param-coercion] - const source_node = await (new FSNodeParam('uid')).consolidate({ - req, getParam: () => req.query.uid - }); - - // TODO: [fs:operation:param-coercion] - const dest_node = await (new FSNodeParam('dest_path')).consolidate({ - req, getParam: () => req.body.dest_path ?? req.body.destination_uid - }); - - // Get user - let user = await get_user({id: await source_node.get('user_id')}); - - const opts = { - source: source_node, - destination_or_parent: dest_node, - dedupe_name: change_name, - overwrite, - user, - }; - - let new_fsentries - try{ - const hl_copy = new HLCopy(); - - const r = await Context.get().sub({ actor: Actor.adapt(user) }).arun(async () => { - return await hl_copy.run({ - ...opts, - actor: Context.get('actor'), - }); - }); - return res.send([r]); - }catch(e){ - console.log(e) - return res.status(400).send(e) - } - } - - // -----------------------------------------------------------------------// - // mkdir - // -----------------------------------------------------------------------// - else if(req.query.operation && req.query.operation === 'mkdir'){ - const {mkdir, uuid2fsentry, get_user, id2path} = require('../helpers') - - // name is required - if(!req.body.name){ - return res.status(400).send({ - error:{ - message: 'Name is required.' - } - }) - } - - // TODO: [fs:operation:param-coercion] - const source_node = await (new FSNodeParam('uid')).consolidate({ - req, getParam: () => req.query.uid - }); - - - // Get user - let user = await get_user({id: await source_node.get('user_id')}); - - // Create new dir and return - try{ - // TODO: [fs:remove-old-methods] - const hl_mkdir = new HLMkdir(); - const r = await Context.get().sub({ actor: Actor.adapt(user) }).arun(async () => { - return await hl_mkdir.run({ - parent: source_node, - path: req.body.name, - overwrite: false, - dedupe_name: req.body.dedupe_name ?? false, - user, - actor: Context.get('actor'), - }); - }); - const newdir_node = await req.fs.node(new NodeUIDSelector(r.uid)); - return res.send(await sign_file( - await newdir_node.get('entry'), 'write')); - }catch(e){ - console.log(e) - return res.status(400).send(e); - } - } - - // -----------------------------------------------------------------------// - // Trash - // -----------------------------------------------------------------------// - if(req.query.operation && req.query.operation === 'trash'){ - const {validate_fsentry_name, convert_path_to_fsentry, uuid2fsentry, get_user, id2path, id2uuid} = require('../helpers') - const _path = require('path'); - const mime = require('mime-types'); - - // Get fsentry - const fs = req.services.get('filesystem'); - - // TODO: [fs:move-FSNodeParam] - const node = await (new FSNodeParam('path')).consolidate({ - req, getParam: () => req.query.uid - }); - - // Get user - // TODO: [avoid-database-user-id] - let user = await get_user({id: await node.get('user_id')}); - - // metadata for trashed file - const new_name = await node.get('uid'); - const metadata = { - original_name: await node.get('name'), - original_path: await node.get('path'), - trashed_ts: Math.round(Date.now() / 1000), - }; - - // Get Trash fsentry - const trash = await fs.node( - new NodePathSelector('/' + user.username + '/Trash') - ); - // let trash_path = '/' + user.username + '/Trash'; - // let trash = await convert_path_to_fsentry(trash_path); - - console.log('what is trash?', trash); - - const hl_move = new HLMove(); - await Context.get().sub({ actor: Actor.adapt(user) }).arun(async () => { - await hl_move.run({ - source: node, - destination_or_parent: trash, - // TODO: [fs:decouple-user] - user, - actor: Context.get('actor'), - new_name: new_name, - new_metadata: metadata, - }); - }); - - // No Trash? - if(!trash){ - return res.status(400).send({ - error:{ - message: 'No Trash directory found.' - } - }) - } - - return res.status(200).send({ - message: 'Item trashed' - }) - } - // -----------------------------------------------------------------------// - // Rename - // -----------------------------------------------------------------------// - if(req.query.operation && req.query.operation === 'rename'){ - const {validate_fsentry_name, uuid2fsentry, get_app, id2path} = require('../helpers') - const _path = require('path'); - const mime = require('mime-types'); - - // new_name validation - try{ - validate_fsentry_name(req.body.new_name) - }catch(e){ - return res.status(400).send({ - error:{ - message: e.message - } - }); - } - - // Get fsentry - let fsentry = await uuid2fsentry(req.query.uid); - - // Not found - if(fsentry === false){ - return res.status(400).send({ - error:{ - message: 'No entry found with this uid' - } - }) - } - - // Immutable? - if(fsentry.immutable){ - return res.status(400).send({ - error:{ - message: 'Immutable: cannot rename.' - } - }) - } - - let res1; - - // parent is root - if(fsentry.parent_uid === null){ + return; + } try{ - res1 = await db.read( - `SELECT uuid FROM fsentries WHERE parent_uid IS NULL AND name = ? AND id != ? LIMIT 1`, - [ - //name - req.body.new_name, - fsentry.id, - ]); + validate_signature_auth(req.body.destination_write_url, 'write', { + uid: req.body.destination_uid, + }); }catch(e){ - console.log(e) + res.status(403).send(e); + return; } - } - // parent is regular dir - else{ - res1 = await db.read( - `SELECT uuid FROM fsentries WHERE parent_uid = ? AND name = ? AND id != ? LIMIT 1`, - [ - //parent_uid - fsentry.parent_uid, - //name - req.body.new_name, - fsentry.id, - ]); - } - if(res1[0]){ - return res.status(400).send({ - error:{ - message: 'An entry with the same name exists under target path.' - } - }) - } - - // old path - const old_path = await id2path(fsentry.id); - - // update `name` - await db.write( - `UPDATE fsentries SET name = ? WHERE id = ?`, - [req.body.new_name, fsentry.id] - ) - - // new path - const new_path = _path.join(_path.dirname(old_path), req.body.new_name); - - // associated_app - let associated_app; - if(fsentry.associated_app_id){ - const app = await get_app({id: fsentry.associated_app_id}) - // remove some privileged information - delete app.id; - delete app.approved_for_listing; - delete app.approved_for_opening_items; - delete app.godmode; - delete app.owner_user_id; - // add to array - associated_app = app; - }else{ - associated_app = {}; - } - - // send the fsentry of the new object created - const contentType = mime.contentType(req.body.new_name) - const return_obj = { - uid: fsentry.uuid, - name: req.body.new_name, - is_dir: fsentry.is_dir, - path: new_path, - old_path: old_path, - type: contentType ? contentType : null, - associated_app: associated_app, - original_client_socket_id: req.body.original_client_socket_id, - }; - - // send realtime success msg to client - let socketio = require('../socketio.js').getio(); - if(socketio){ - socketio.to(fsentry.user_id).emit('item.renamed', return_obj) - } - - return res.send(return_obj); - } - - // -----------------------------------------------------------------------// - // Delete - // -----------------------------------------------------------------------// - if(req.query.operation && req.query.operation === 'delete'){ - const {get_user, uuid2fsentry, id2path} = require('../helpers') - const _path = require('path'); - const mime = require('mime-types'); - - // TODO: [fs:operation:param-coercion] - const source_node = await (new FSNodeParam('uid')).consolidate({ - req, getParam: () => req.query.uid - }); - - const user = await get_user({id: await source_node.get('user_id')}); - - // Delete - try{ - const hl_remove = new HLRemove(); - await Context.get().sub({ actor: Actor.adapt(user) }).arun(async () => { - await hl_remove.run({ - target: source_node, - user, - actor: Context.get('actor'), + try { + return await (new FSNodeParam('dest_path')).consolidate({ + req, getParam: () => req.body.dest_path ?? req.body.destination_uid }); - }); - }catch(error){ - console.log(error) - res.status(400).send(error); + } catch (e) { + res.status(500).send('Internal Server Error'); + } } + }; - // Send success msg - return res.send(); + const writeFile_handlers = require('./writeFile/writeFile_handlers.js'); + + let operation = req.query.operation ?? 'write'; + // Responding with an error here would typically be better, + // but it would cause a regression for apps. + if ( ! writeFile_handlers.hasOwnProperty(operation) ) { + operation = 'write'; } - // -----------------------------------------------------------------------// - // Write - // -----------------------------------------------------------------------// - else{ - // modules - const {uuid2fsentry, id2path} = require('../helpers') - const _path = require('path'); - - // Check if files were uploaded - if(!req.files) - return res.status(400).send('No files uploaded'); - - // Get fsentry - let fsentry, dirname; - let node; - - try{ - node = await req.fs.node(new NodeUIDSelector(req.query.uid)); - dirname = (await node.get('type') !== TYPE_DIRECTORY - ? _path.dirname.bind(_path) : a=>a)(await node.get('path')); - }catch(e){ - console.log(e) - req.__error_source = e; - return res.status(500).send(e); - } - - const user = await (async () => { - const { get_user } = require('../helpers'); - const user_id = await node.get('user_id') - return await get_user({ id: user_id }); - })(); - Context.set('user', user); - - const dirNode = await req.fs.node(new NodePathSelector(dirname)); - - const actor = Actor.adapt(user); - - const context = Context.get().sub({ - actor, user, + console.log('\x1B[36;1mwriteFile: ' + req.query.operation + '\x1B[0m'); + const node = await (new FSNodeParam('uid')).consolidate({ + req, getParam: () => req.query.uid + }); + const user = await get_user({id: await node.get('user_id')}); + const actor = Actor.adapt(user); + + return await Context.get().sub({ + actor: Actor.adapt(user), user, + }).arun(async () => { + return await writeFile_handlers[operation]({ + api: writeFile_handler_api, + req, res, actor, + node, }); - - log.noticeme('writeFile: ' + context.describe()); - - // Upload files one by one - const returns = []; - for ( const uploaded_file of req.files ) { - try{ - await context.arun(async () => { - const hl_write = new HLWrite(); - const ret_obj = await hl_write.run({ - destination_or_parent: dirNode, - specified_name: await node.get('type') === TYPE_DIRECTORY - ? req.body.name : await node.get('name'), - fallback_name: uploaded_file.originalname, - overwrite: true, - user: user, - actor, - - file: uploaded_file, - }); - - // add signature to object - ret_obj.signature = await sign_file(ret_obj, 'write'); - - // send results back to app - returns.push(ret_obj); - }); - }catch(error){ - req.__error_source = error; - console.log(error) - return res.contentType('application/json').status(500).send(error); - } - } - - if ( returns.length === 1 ) { - return res.send(returns[0]); - } - - return res.send(returns); - } + }); }); diff --git a/src/backend/src/routers/writeFile/copy.js b/src/backend/src/routers/writeFile/copy.js new file mode 100644 index 0000000000..445399f735 --- /dev/null +++ b/src/backend/src/routers/writeFile/copy.js @@ -0,0 +1,32 @@ +const { HLCopy } = require('../../filesystem/hl_operations/hl_copy'); + +module.exports = async function writeFile_handle_copy ({ + api, + req, res, actor, node, +}) { + + // check if destination_write_url provided + + // check if destination_write_url is valid + const dest_node = await api.get_dest_node(); + if ( ! dest_node ) return; + + const overwrite = req.body.overwrite ?? false; + const change_name = req.body.auto_rename ?? false; + + const opts = { + source: node, + destination_or_parent: dest_node, + dedupe_name: change_name, + overwrite, + user: actor.type.user, + }; + + const hl_copy = new HLCopy(); + + const r = await hl_copy.run({ + ...opts, + actor, + }); + return res.send([r]); +} diff --git a/src/backend/src/routers/writeFile/delete.js b/src/backend/src/routers/writeFile/delete.js new file mode 100644 index 0000000000..7b173c3840 --- /dev/null +++ b/src/backend/src/routers/writeFile/delete.js @@ -0,0 +1,17 @@ +const { HLRemove } = require("../../filesystem/hl_operations/hl_remove"); + +module.exports = async function writeFile_handle_delete ({ + req, res, actor, node, +}) { + // Delete + const hl_remove = new HLRemove(); + await hl_remove.run({ + target: node, + user: actor.type.user, + actor, + }); + + // Send success msg + return res.send(); +} + diff --git a/src/backend/src/routers/writeFile/mkdir.js b/src/backend/src/routers/writeFile/mkdir.js new file mode 100644 index 0000000000..a6ac3cb25b --- /dev/null +++ b/src/backend/src/routers/writeFile/mkdir.js @@ -0,0 +1,26 @@ +const { HLMkdir } = require("../../filesystem/hl_operations/hl_mkdir"); +const { NodeUIDSelector } = require("../../filesystem/node/selectors"); +const { sign_file } = require("../../helpers"); + +module.exports = async function writeFile_handle_mkdir ({ + req, res, actor, node +}) { + if( ! req.body.name ) return res.status(400).send({ + error:{ + message: 'Name is required.' + } + }) + + const hl_mkdir = new HLMkdir(); + const r = await hl_mkdir.run({ + parent: node, + path: req.body.name, + overwrite: false, + dedupe_name: req.body.dedupe_name ?? false, + user: actor.type.user, + actor, + }); + + const newdir_node = await req.fs.node(new NodeUIDSelector(r.uid)); + return res.send(await sign_file(await newdir_node.get('entry'), 'write')); +}; diff --git a/src/backend/src/routers/writeFile/move.js b/src/backend/src/routers/writeFile/move.js new file mode 100644 index 0000000000..1f2f13ff43 --- /dev/null +++ b/src/backend/src/routers/writeFile/move.js @@ -0,0 +1,41 @@ +const { HLMove } = require("../../filesystem/hl_operations/hl_move"); + +module.exports = async function writeFile_handle_move ({ + api, + req, res, actor, node, +}) { + // check if destination_write_url provided + if(!req.body.destination_write_url){ + return res.status(400).send({ + error:{ + message: 'No destination specified.' + } + }) + } + + const dest_node = await api.get_dest_node(); + if ( ! dest_node ) return; + + const hl_move = new HLMove(); + + const opts = { + user: actor.type.user, + source: node, + destination_or_parent: dest_node, + overwrite: req.body.overwrite ?? false, + new_name: req.body.new_name, + new_metadata: req.body.new_metadata, + create_missing_parents: req.body.create_missing_parents, + }; + + const r = await hl_move.run({ + ...opts, + actor, + }); + + return res.send({ + ...r.moved, + old_path: r.old_path, + new_path: r.moved.path, + }); +} diff --git a/src/backend/src/routers/writeFile/rename.js b/src/backend/src/routers/writeFile/rename.js new file mode 100644 index 0000000000..3574b48d85 --- /dev/null +++ b/src/backend/src/routers/writeFile/rename.js @@ -0,0 +1,54 @@ +const mime = require('mime-types'); +const { validate_fsentry_name } = require("../../helpers"); +const { DB_WRITE } = require('../../services/database/consts'); + +module.exports = async function writeFile_handle_rename ({ + req, res, node, +}) { + const new_name = req.body.new_name; + + try { + validate_fsentry_name(new_name); + } catch(e) { + return res.status(400).send({ + error:{ + message: e.message + } + }); + } + + if ( await node.get('immutable') ) { + return res.status(400).send({ + error:{ + message: 'Immutable: cannot rename.' + } + }) + } + + if ( await node.isUserDirectory() || await node.isRoot ) { + return res.status(403).send({ + error:{ + message: 'Not allowed to rename this item via writeFile.' + } + }) + } + + const old_path = await node.get('path'); + + const db = req.services.get('database').get(DB_WRITE, 'writeFile:rename'); + const mysql_id = await node.get('mysql-id'); + await db.write( + `UPDATE fsentries SET name = ? WHERE id = ?`, + [new_name, mysql_id] + ); + + const contentType = mime.contentType(req.body.new_name) + const return_obj = { + ...await node.getSafeEntry(), + old_path, + type: contentType ? contentType : null, + original_client_socket_id: req.body.original_client_socket_id, + }; + + return res.send(return_obj); +} diff --git a/src/backend/src/routers/writeFile/trash.js b/src/backend/src/routers/writeFile/trash.js new file mode 100644 index 0000000000..f12a8cc39c --- /dev/null +++ b/src/backend/src/routers/writeFile/trash.js @@ -0,0 +1,44 @@ +const { HLMove } = require("../../filesystem/hl_operations/hl_move"); +const { NodePathSelector } = require("../../filesystem/node/selectors"); + +module.exports = async function writeFile_handle_trash ({ + req, res, actor, node, +}) { + // metadata for trashed file + const new_name = await node.get('uid'); + const metadata = { + original_name: await node.get('name'), + original_path: await node.get('path'), + trashed_ts: Math.round(Date.now() / 1000), + }; + + // Get Trash fsentry + const fs = req.services.get('filesystem'); + const trash = await fs.node( + new NodePathSelector('/' + actor.type.user.username + '/Trash') + ); + + // No Trash? + if(!trash){ + return res.status(400).send({ + error:{ + message: 'No Trash directory found.' + } + }) + } + + const hl_move = new HLMove(); + await hl_move.run({ + source: node, + destination_or_parent: trash, + user: actor.type.user, + actor, + new_name: new_name, + new_metadata: metadata, + }); + + + return res.status(200).send({ + message: 'Item trashed' + }) +}; diff --git a/src/backend/src/routers/writeFile/write.js b/src/backend/src/routers/writeFile/write.js new file mode 100644 index 0000000000..0b90ae213d --- /dev/null +++ b/src/backend/src/routers/writeFile/write.js @@ -0,0 +1,64 @@ +const { TYPE_DIRECTORY } = require("../../filesystem/FSNodeContext"); +const { HLWrite } = require("../../filesystem/hl_operations/hl_write"); +const { NodePathSelector } = require("../../filesystem/node/selectors"); +const _path = require('path'); +const { sign_file } = require("../../helpers"); + +module.exports = async function writeFile_handle_write ({ + req, res, actor, node, +}) { + + // Check if files were uploaded + if(!req.files) { + return res.status(400).send('No files uploaded'); + } + + // Get fsentry + let dirname; + + try{ + dirname = (await node.get('type') !== TYPE_DIRECTORY + ? _path.dirname.bind(_path) : a=>a)(await node.get('path')); + }catch(e){ + console.log(e) + req.__error_source = e; + return res.status(500).send(e); + } + + const dirNode = await req.fs.node(new NodePathSelector(dirname)); + + // Upload files one by one + const returns = []; + for ( const uploaded_file of req.files ) { + try{ + const hl_write = new HLWrite(); + const ret_obj = await hl_write.run({ + destination_or_parent: dirNode, + specified_name: await node.get('type') === TYPE_DIRECTORY + ? req.body.name : await node.get('name'), + fallback_name: uploaded_file.originalname, + overwrite: true, + user: actor.type.user, + actor, + + file: uploaded_file, + }); + + // add signature to object + ret_obj.signature = await sign_file(ret_obj, 'write'); + + // send results back to app + returns.push(ret_obj); + }catch(error){ + req.__error_source = error; + console.log(error) + return res.contentType('application/json').status(500).send(error); + } + } + + if ( returns.length === 1 ) { + return res.send(returns[0]); + } + + return res.send(returns); +}; diff --git a/src/backend/src/routers/writeFile/writeFile_handlers.js b/src/backend/src/routers/writeFile/writeFile_handlers.js new file mode 100644 index 0000000000..aad004431c --- /dev/null +++ b/src/backend/src/routers/writeFile/writeFile_handlers.js @@ -0,0 +1,9 @@ +module.exports = { + move: require('./move'), + copy: require('./copy'), + mkdir: require('./mkdir'), + trash: require('./trash'), + delete: require('./delete'), + rename: require('./rename'), + write: require('./write'), +}; diff --git a/src/backend/src/services/AnomalyService.js b/src/backend/src/services/AnomalyService.js index 0c684e3a68..7fa51a3351 100644 --- a/src/backend/src/services/AnomalyService.js +++ b/src/backend/src/services/AnomalyService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"openai-completion","model":"gpt-4o"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -18,9 +19,26 @@ */ const BaseService = require("./BaseService"); +// Symbol used to indicate a denial of service instruction in anomaly handling. const DENY_SERVICE_INSTRUCTION = Symbol('DENY_SERVICE_INSTRUCTION'); + +/** +* @class AnomalyService +* @extends BaseService +* @description The AnomalyService class is responsible for managing and processing anomaly detection types and configurations. +* It allows the registration of different types with associated handlers, enabling the detection of anomalies based on specified criteria. +*/ class AnomalyService extends BaseService { + /** + * AnomalyService class that extends BaseService and provides methods + * for registering anomaly types and handling incoming data for those anomalies. + * + * The register method allows the registration of different anomaly types + * and their respective configurations, including custom handlers for data + * evaluation. It supports two modes of operation: a direct handler or + * a threshold-based evaluation. + */ _construct () { this.types = {}; } @@ -39,6 +57,16 @@ class AnomalyService extends BaseService { } this.types[type] = type_instance; } + /** + * Registers a new type with the service, including its configuration and handler. + * + * @param {string} type - The name of the type to register. + * @param {Object} config - The configuration object for the type. + * @param {Function} [config.handler] - An optional handler function for the type. + * @param {number} [config.high] - An optional threshold value; triggers the handler if exceeded. + * + * @returns {void} + */ async note (id, data) { const type = this.types[id]; if ( ! type ) return; diff --git a/src/backend/src/services/AppIconService.js b/src/backend/src/services/AppIconService.js new file mode 100644 index 0000000000..d369d58dde --- /dev/null +++ b/src/backend/src/services/AppIconService.js @@ -0,0 +1,162 @@ +const { HLWrite } = require("../filesystem/hl_operations/hl_write"); +const { LLMkdir } = require("../filesystem/ll_operations/ll_mkdir"); +const { LLRead } = require("../filesystem/ll_operations/ll_read"); +const { NodePathSelector } = require("../filesystem/node/selectors"); +const { Endpoint } = require("../util/expressutil"); +const { buffer_to_stream } = require("../util/streamutil"); +const BaseService = require("./BaseService"); + +const ICON_SIZES = [16,32,64,128,256,512]; + +/** + * AppIconService handles icon generation and serving for apps. + * + * This is done by listening to the `app.new-icon` event which is + * dispatched by AppES. `sharp` is used to resize the images to + * pre-selected sizees in the `ICON_SIZES` constant defined above. + * + * Icons are stored in and served from the `/system/app_icons` + * directory. If the system user does not have this directory, + * it will be created in the consolidation boot phase after + * UserService emits the `user.system-user-ready` event on the + * service container event bus. + */ +class AppIconService extends BaseService { + static MODULES = { + sharp: require('sharp'), + } + + /** + * AppIconService listens to this event to register the + * endpoint /app-icon/:app_uid/:size which serves the + * app icon at the requested size. + */ + async ['__on_install.routes'] (_, { app }) { + Endpoint({ + route: '/app-icon/:app_uid/:size', + methods: ['GET'], + handler: async (req, res) => { + // Validate parameters + let { app_uid, size } = req.params; + if ( ! ICON_SIZES.includes(Number(size)) ) { + res.status(400).send('Invalid size'); + return; + } + if ( ! app_uid.startsWith('app-') ) { + app_uid = `app-${app_uid}`; + } + + // Get icon file node + const dir_app_icons = await this.get_app_icons(); + const node = await dir_app_icons.getChild(`${app_uid}-${size}.png`); + await node.fetchEntry(); + + const svc_su = this.services.get('su'); + const ll_read = new LLRead(); + const stream = await ll_read.run({ + fsNode: node, + actor: await svc_su.get_system_actor(), + }); + + res.set('Content-Type', 'image/png'); + stream.pipe(res); + }, + }).attach(app); + } + + /** + * Returns an FSNodeContext instance for the app icons + * directory. + */ + async get_app_icons () { + if ( this.dir_app_icons ) { + return this.dir_app_icons; + } + + const svc_fs = this.services.get('filesystem'); + const dir_app_icons = await svc_fs.node( + new NodePathSelector('/system/app_icons') + ); + + this.dir_app_icons = dir_app_icons; + } + + /** + * AppIconService listens to this event to create the + * `/system/app_icons` directory if it does not exist, + * and then to register the event listener for `app.new-icon`. + */ + async ['__on_user.system-user-ready'] () { + const svc_su = this.services.get('su'); + const svc_fs = this.services.get('filesystem'); + const svc_user = this.services.get('user'); + + const dir_system = await svc_user.get_system_dir(); + + // Ensure app icons directory exists + const dir_app_icons = await svc_fs.node( + new NodePathSelector('/system/app_icons') + ); + if ( ! await dir_app_icons.exists() ) { + const ll_mkdir = new LLMkdir(); + await ll_mkdir.run({ + parent: dir_system, + name: 'app_icons', + actor: await svc_su.get_system_actor(), + }); + } + this.dir_app_icons = dir_app_icons; + + // Listen for new app icons + const svc_event = this.services.get('event'); + svc_event.on('app.new-icon', async (_, data) => { + // Writing icons as the system user + const icon_jobs = []; + for ( const size of ICON_SIZES ) { + icon_jobs.push((async () => { + await svc_su.sudo(async () => { + const filename = `${data.app_uid}-${size}.png`; + console.log('FILENAME', filename); + const data_url = data.data_url; + const base64 = data_url.split(',')[1]; + const input = Buffer.from(base64, 'base64'); + + // NOTE: A stream would be more ideal than a buffer here + // but we have no way of knowing the output size + // before we finish processing the image. + const output = await this.modules.sharp(input) + .resize(size) + .png() + .toBuffer(); + + const sys_actor = await svc_su.get_system_actor(); + const hl_write = new HLWrite(); + await hl_write.run({ + destination_or_parent: dir_app_icons, + specified_name: filename, + overwrite: true, + actor: sys_actor, + user: sys_actor.type.user, + no_thumbnail: true, + file: { + size: output.length, + name: filename, + mimetype: 'image/png', + type: 'image/png', + stream: buffer_to_stream(output), + }, + }); + }) + })()); + } + await Promise.all(icon_jobs); + }); + } + + async _init () { + } +} + +module.exports = { + AppIconService, +}; diff --git a/src/backend/src/services/AppInformationService.js b/src/backend/src/services/AppInformationService.js index b473de1228..9bbaf95cd7 100644 --- a/src/backend/src/services/AppInformationService.js +++ b/src/backend/src/services/AppInformationService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"xai"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -16,13 +17,24 @@ * You should have received a copy of the GNU Affero General Public License * along with this program. If not, see . */ -const { asyncSafeSetInterval } = require("../util/promise"); -const { MINUTE, SECOND } = require("../util/time"); +const { asyncSafeSetInterval } = require('@heyputer/putility').libs.promise; +const { MINUTE, SECOND } = require("@heyputer/putility").libs.time; const { origin_from_url } = require("../util/urlutil"); const { DB_READ } = require("./database/consts"); const uuidv4 = require('uuid').v4; + +/** +* @class AppInformationService +* @description +* The AppInformationService class manages application-related information, +* including caching, statistical data, and tags for applications within the Puter ecosystem. +* It provides methods for refreshing application data, managing app statistics, +* and handling tags associated with apps. This service is crucial for maintaining +* up-to-date information about applications, facilitating features like app listings, +* recent apps, and tag-based app discovery. +*/ class AppInformationService { constructor ({ services }) { this.services = services; @@ -37,33 +49,87 @@ class AppInformationService { // await new Promise(rslv => setTimeout(rslv, 500)) await this._refresh_app_cache(); + /** + * Refreshes the application cache by querying the database for all apps and updating the key-value store. + * + * This method is called periodically to ensure that the in-memory cache reflects the latest + * state from the database. It uses the 'database' service to fetch app data and then updates + * multiple cache entries for quick lookups by name, ID, and UID. + * + * @async + */ asyncSafeSetInterval(async () => { this._refresh_app_cache(); }, 30 * 1000); await this._refresh_app_stats(); + /** + * Refreshes the cache of recently opened apps. + * This method updates the 'recent' collection with the UIDs of apps sorted by their most recent timestamp. + * + * @async + * @returns {Promise} A promise that resolves when the cache has been refreshed. + */ asyncSafeSetInterval(async () => { this._refresh_app_stats(); }, 120 * 1000); // This stat is more expensive so we don't update it as often await this._refresh_app_stat_referrals(); + /** + * Refreshes the app referral statistics. + * This method is computationally expensive and thus runs less frequently. + * It queries the database for user counts referred by each app's origin URL. + * + * @async + */ asyncSafeSetInterval(async () => { this._refresh_app_stat_referrals(); }, 15 * MINUTE); await this._refresh_recent_cache(); + /** + * Refreshes the recent cache by updating the list of recently added or updated apps. + * This method fetches all app data, filters for approved apps, sorts them by timestamp, + * and updates the `this.collections.recent` array with the UIDs of the most recent 50 apps. + * + * @async + * @private + */ asyncSafeSetInterval(async () => { this._refresh_recent_cache(); }, 120 * 1000); await this._refresh_tags(); + /** + * Refreshes the tags cache by iterating through all approved apps, + * extracting their tags, and organizing them into a structured format. + * This method updates the `this.tags` object with the latest tag information. + * + * @async + * @method + * @memberof AppInformationService + */ asyncSafeSetInterval(async () => { this._refresh_tags(); } , 120 * 1000); })(); } + + /** + * Retrieves and returns statistical data for a specific application. + * + * This method fetches various metrics such as the number of times the app has been opened, + * the count of unique users who have opened the app, and the number of referrals attributed to the app. + * It uses cached data where available to improve performance. + * + * @param {string} app_uid - The unique identifier for the application. + * @returns {Promise} An object containing: + * - {number} open_count - The total number of times the app has been opened. + * - {number} user_count - The count of unique users who have opened the app. + * - {number|null} referral_count - The number of referrals, or null if the data is not available or too expensive to retrieve. + */ async get_stats (app_uid) { const db = this.services.get('database').get(DB_READ, 'apps'); @@ -100,20 +166,43 @@ class AppInformationService { }; } + + /** + * Retrieves various statistics for a given app. + * + * This method fetches the open count, user count, and referral count for an app identified by its UID. + * It uses cached values where available to improve performance, but will query the database if necessary. + * + * @param {string} app_uid - The unique identifier of the app for which to retrieve stats. + * @returns {Promise} An object containing: + * - {number} open_count - Total number of times the app was opened. + * - {number} user_count - Number of unique users who opened the app. + * - {number|null} referral_count - Number of referrals attributed to the app. This value might not be reported if not cached. + */ async _refresh_app_cache () { this.log.tick('refresh app cache'); const db = this.services.get('database').get(DB_READ, 'apps'); let apps = await db.read('SELECT * FROM apps'); - for (let index = 0; index < apps.length; index++) { - const app = apps[index]; + for ( const app of apps ) { kv.set('apps:name:' + app.name, app); kv.set('apps:id:' + app.id, app); kv.set('apps:uid:' + app.uid, app); } } + + /** + * Refreshes the application cache by querying the database for all apps and updating the key-value store. + * + * @async + * @returns {Promise} A promise that resolves when the cache refresh operation is complete. + * + * @notes + * - This method logs a tick event for performance monitoring. + * - It populates the cache with app data indexed by name, id, and uid. + */ async _refresh_app_stats () { this.log.tick('refresh app stats'); @@ -144,6 +233,13 @@ class AppInformationService { } } + + /** + * Refreshes the cache of app statistics including open and user counts. + * This method updates the cache every 120 seconds to ensure data freshness. + * + * @async + */ async _refresh_app_stat_referrals () { this.log.tick('refresh app stat referrals'); @@ -152,9 +248,6 @@ class AppInformationService { const apps = await db.read(`SELECT uid, index_url FROM apps`); for ( const app of apps ) { - const sql = - `SELECT COUNT(id) AS referral_count FROM user WHERE referrer = ?`; - const origin = origin_from_url(app.index_url); // only count the referral if the origin hashes to the app's uid @@ -176,9 +269,19 @@ class AppInformationService { this.log.info('DONE refresh app stat referrals'); } + + /** + * Updates the cache with recently updated apps. + * + * @description This method refreshes the cache containing the most recently updated applications. + * It fetches all app UIDs, retrieves the corresponding app data, filters for approved apps, + * sorts them by timestamp in descending order, and updates the 'recent' collection with + * the UIDs of the top 50 most recent apps. + * + * @returns {Promise} Resolves when the cache has been updated. + */ async _refresh_recent_cache () { const app_keys = kv.keys(`apps:uid:*`); - // console.log('APP KEYS', app_keys); let apps = []; for ( const key of app_keys ) { @@ -194,9 +297,18 @@ class AppInformationService { this.collections.recent = apps.map(app => app.uid).slice(0, 50); } + + /** + * Refreshes the cache of recently added or updated apps. + * + * This method retrieves all apps from the cache, filters for approved listings, + * sorts them by timestamp in descending order, and updates the `recent` collection + * with the UIDs of the most recent 50 apps. + * + * @returns {Promise} + */ async _refresh_tags () { const app_keys = kv.keys(`apps:uid:*`); - // console.log('APP KEYS', app_keys); let apps = []; for ( const key of app_keys ) { @@ -229,6 +341,22 @@ class AppInformationService { this.tags = new_tags; } + + /** + * Deletes an application from the system. + * + * This method performs the following actions: + * - Retrieves the app data from cache or database if not provided. + * - Deletes the app record from the database. + * - Removes the app from all relevant caches (by name, id, and uid). + * - Removes the app from the recent collection if present. + * - Removes the app from any associated tags. + * + * @param {string} app_uid - The unique identifier of the app to be deleted. + * @param {Object} [app] - The app object, if already fetched. If not provided, it will be retrieved. + * @throws {Error} If the app is not found in either cache or database. + * @returns {Promise} A promise that resolves when the app has been successfully deleted. + */ async delete_app (app_uid, app) { const db = this.services.get('database').get(DB_READ, 'apps'); diff --git a/src/backend/src/services/BaseService.js b/src/backend/src/services/BaseService.js index 1b169fae6d..2e74b0d393 100644 --- a/src/backend/src/services/BaseService.js +++ b/src/backend/src/services/BaseService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-params":{"service":"xai"},"ai-refs":["../../doc/contributors/boot-sequence.md"],"ai-commented":{"service":"xai"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -18,11 +19,28 @@ */ const { concepts } = require("@heyputer/putility"); + + +// This is a no-op function that AI is incapable of writing a comment for. +// That said, I suppose it didn't need one anyway. const NOOP = async () => {}; + +/** +* @class BaseService +* @extends concepts.Service +* @description +* BaseService is the foundational class for all services in the Puter backend. +* It provides lifecycle methods like `construct` and `init` that are invoked during +* different phases of the boot sequence. This class ensures that services can be +* instantiated, initialized, and activated in a coordinated manner through +* events emitted by the Kernel. It also manages common service resources like +* logging and error handling, and supports legacy services by allowing +* instantiation after initialization but before consolidation. +*/ class BaseService extends concepts.Service { constructor (service_resources, ...a) { - const { services, config, my_config, name, args } = service_resources; + const { services, config, my_config, name, args, context } = service_resources; super(service_resources, ...a); this.args = args; @@ -30,16 +48,42 @@ class BaseService extends concepts.Service { this.services = services; this.config = my_config; this.global_config = config; + this.context = context; if ( this.global_config.server_id === '' ) { this.global_config.server_id = 'local'; } } + + /** + * Initializes the service with configuration and dependencies. + * This method sets up logging and error handling, and calls a custom `_init` method if defined. + * + * @param {Object} args - Arguments passed to the service for initialization. + * @returns {Promise} A promise that resolves when initialization is complete. + */ async construct () { + console.log('CLASS', this.constructor.name); + const useapi = this.context.get('useapi'); + const use = this._get_merged_static_object('USE'); + for ( const [key, value] of Object.entries(use) ) { + this[key] = useapi.use(value); + } await (this._construct || NOOP).call(this, this.args); } + + /** + * Performs the initialization phase of the service lifecycle. + * This method sets up logging and error handling for the service, + * then calls the service-specific initialization logic if defined. + * + * @async + * @memberof BaseService + * @instance + * @returns {Promise} A promise that resolves when initialization is complete. + */ async init () { const services = this.services; this.log = services.get('log-service').create(this.service_name); @@ -48,6 +92,15 @@ class BaseService extends concepts.Service { await (this._init || NOOP).call(this, this.args); } + + /** + * Handles an event by retrieving the appropriate event handler + * and executing it with the provided arguments. + * + * @param {string} id - The identifier of the event to handle. + * @param {Array} args - The arguments to pass to the event handler. + * @returns {Promise} The result of the event handler execution. + */ async __on (id, args) { const handler = this.__get_event_handler(id); diff --git a/src/backend/src/services/BootScriptService.js b/src/backend/src/services/BootScriptService.js index 4b587ceeec..2cfef92464 100644 --- a/src/backend/src/services/BootScriptService.js +++ b/src/backend/src/services/BootScriptService.js @@ -1,10 +1,29 @@ +// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}} const { Context } = require("../util/context"); const BaseService = require("./BaseService"); + +/** +* @class BootScriptService +* @extends BaseService +* @description The BootScriptService class extends BaseService and is responsible for +* managing and executing boot scripts. It provides methods to handle boot scripts when +* the system is ready and to run individual script commands. +*/ class BootScriptService extends BaseService { static MODULES = { fs: require('fs'), } + /** + * Loads and executes a boot script if specified in the arguments. + * + * This method reads the provided boot script file, parses it, and runs the script using the `run_script` method. + * If no boot script is specified in the arguments, the method returns immediately. + * + * @async + * @function + * @returns {Promise} + */ async ['__on_boot.ready'] () { const args = Context.get('args'); if ( ! args['boot-script'] ) return; @@ -17,6 +36,17 @@ class BootScriptService extends BaseService { await this.run_script(boot_json); } + + /** + * Executes a series of commands defined in a JSON boot script. + * + * This method processes each command in the boot_json array. + * If the command is recognized within the predefined scope, it will be executed. + * If not, an error is thrown. + * + * @param {Array} boot_json - An array of commands to execute. + * @throws {Error} Thrown if an unknown command is encountered. + */ async run_script (boot_json) { const scope = { runner: 'boot-script', @@ -26,8 +56,7 @@ class BootScriptService extends BaseService { } }; - for ( let i=0 ; i < boot_json.length ; i++ ) { - const statement = boot_json[i]; + for ( const statement of boot_json ) { const [cmd, ...args] = statement; if ( ! scope[cmd] ) { throw new Error(`Unknown command: ${cmd}`); diff --git a/src/backend/src/services/CleanEmailService.js b/src/backend/src/services/CleanEmailService.js index 5fe2919287..1d58f8f966 100644 --- a/src/backend/src/services/CleanEmailService.js +++ b/src/backend/src/services/CleanEmailService.js @@ -1,6 +1,15 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const { can } = require("../util/langutil"); const BaseService = require("./BaseService"); + +/** +* CleanEmailService - A service class for cleaning and validating email addresses +* Handles email normalization by applying provider-specific rules (e.g. Gmail's dot-insensitivity), +* manages subaddressing (plus addressing), and validates against blocked domains. +* Extends BaseService to integrate with the application's service infrastructure. +* @extends BaseService +*/ class CleanEmailService extends BaseService { static NAMED_RULES = { // For some providers, dots don't matter @@ -54,6 +63,12 @@ class CleanEmailService extends BaseService { static DOMAIN_NONDISTINCT = { 'googlemail.com': 'gmail.com', } + /** + * Maps non-distinct email domains to their canonical equivalents. + * For example, 'googlemail.com' is mapped to 'gmail.com' since they + * represent the same email service. + * @type {Object.} + */ _construct () { this.named_rules = this.constructor.NAMED_RULES; this.providers = this.constructor.PROVIDERS; @@ -62,6 +77,16 @@ class CleanEmailService extends BaseService { } clean (email) { + /** + * Cleans an email address by applying provider-specific rules and standardizations + * @param {string} email - The email address to clean + * @returns {string} The cleaned email address with applied rules and standardizations + * + * Splits email into local and domain parts, applies provider-specific rules like: + * - Removing dots for certain providers (Gmail, iCloud) + * - Handling subaddressing (removing +suffix) + * - Normalizing domains (e.g. googlemail.com -> gmail.com) + */ const eml = (() => { const [local, domain] = email.split('@'); return { local, domain }; @@ -101,6 +126,15 @@ class CleanEmailService extends BaseService { return eml.local + '@' + eml.domain; } + + /** + * Validates an email address against blocked domains and custom validation rules + * @param {string} email - The email address to validate + * @returns {Promise} True if email is valid, false if blocked or invalid + * @description First cleans the email, then checks against blocked domains from config. + * Emits 'email.validate' event to allow custom validation rules. Event handlers can + * set event.allow=false to reject the email. + */ async validate (email) { email = this.clean(email); const config = this.global_config; diff --git a/src/backend/src/services/ClientOperationService.js b/src/backend/src/services/ClientOperationService.js index 58c93deba9..c670f61ca2 100644 --- a/src/backend/src/services/ClientOperationService.js +++ b/src/backend/src/services/ClientOperationService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"openai-completion","model":"gpt-4o"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -18,7 +19,14 @@ */ const { Context } = require("../util/context"); +// Key for tracing operations in the context, used for logging and tracking. const CONTEXT_KEY = Context.make_context_key('operation-trace'); +/** +* Class representing a tracker for individual client operations. +* The ClientOperationTracker class is designed to handle the metadata +* and attributes associated with each operation, allowing for better +* management and organization of client data during processing. +*/ class ClientOperationTracker { constructor (parameters) { this.name = parameters.name || 'untitled'; @@ -29,11 +37,26 @@ class ClientOperationTracker { } } + +/** +* Class representing the ClientOperationService, which manages the +* operations related to client interactions. It provides methods to +* add new operations and handle their associated client operation +* trackers, ensuring efficient management and tracking of client-side +* operations during their lifecycle. +*/ class ClientOperationService { constructor ({ services }) { this.operations_ = []; } + + /** + * Adds a new operation to the service by creating a ClientOperationTracker instance. + * + * @param {Object} parameters - The parameters for the new operation. + * @returns {Promise} A promise that resolves to the created ClientOperationTracker instance. + */ async add_operation (parameters) { const tracker = new ClientOperationTracker(parameters); diff --git a/src/backend/src/services/CommandService.js b/src/backend/src/services/CommandService.js index b944f7a55e..00bd156e5f 100644 --- a/src/backend/src/services/CommandService.js +++ b/src/backend/src/services/CommandService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"claude"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -18,15 +19,36 @@ */ const BaseService = require("./BaseService"); + +/** +* Represents a Command class that encapsulates command execution functionality. +* Each Command instance contains a specification (spec) that defines its ID, +* name, description, handler function, and optional argument completer. +* The class provides methods for executing commands and handling command +* argument completion. +*/ class Command { constructor(spec) { this.spec_ = spec; } + + /** + * Gets the unique identifier for this command + * @returns {string} The command's ID as specified in the constructor + */ get id() { return this.spec_.id; } + + /** + * Executes the command with given arguments and logging + * @param {Array} args - Command arguments to pass to the handler + * @param {Object} [log=console] - Logger object for output, defaults to console + * @returns {Promise} + * @throws {Error} Logs any errors that occur during command execution + */ async execute(args, log) { log = log ?? console; const { id, name, description, handler } = this.spec_; @@ -46,10 +68,30 @@ class Command { } } + +/** +* CommandService class manages the registration, execution, and handling of commands in the Puter system. +* Extends BaseService to provide command-line interface functionality. Maintains a collection of Command +* objects, supports command registration with namespaces, command execution with arguments, and provides +* command lookup capabilities. Includes built-in help command functionality. +* @extends BaseService +*/ class CommandService extends BaseService { + /** + * Service for managing and executing commands in the system. + * Extends BaseService to provide command registration, execution and lookup functionality. + * Commands are stored internally with unique IDs and can be executed with arguments. + * Built-in 'help' command is registered during initialization. + */ async _construct () { this.commands_ = []; } + /** + * Initializes the command service's internal state + * Called during service construction to set up the empty commands array + * @private + * @returns {Promise} + */ async _init () { this.commands_.push(new Command({ id: 'help', @@ -64,6 +106,7 @@ class CommandService extends BaseService { } registerCommands(serviceName, commands) { + if ( ! this.log ) process.exit(1); for (const command of commands) { this.log.info(`registering command ${serviceName}:${command.id}`); this.commands_.push(new Command({ @@ -73,6 +116,14 @@ class CommandService extends BaseService { } } + + /** + * Executes a command with the given arguments and logging context + * @param {string[]} args - Array of command arguments where first element is command name + * @param {Object} log - Logger object for output (defaults to console if not provided) + * @returns {Promise} + * @throws {Error} If command execution fails + */ async executeCommand(args, log) { const [commandName, ...commandArgs] = args; const command = this.commands_.find(c => c.spec_.id === commandName); @@ -80,17 +131,37 @@ class CommandService extends BaseService { log.error(`unknown command: ${commandName}`); return; } + /** + * Executes a command with the given arguments in a global context + * @param {string[]} args - Array of command arguments where first element is command name + * @param {Object} log - Logger object for output + * @returns {Promise} + * @throws {Error} If command execution fails + */ await globalThis.root_context.arun(async () => { await command.execute(commandArgs, log); }); } + + /** + * Executes a raw command string by splitting it into arguments and executing the command + * @param {string} text - Raw command string to execute + * @param {object} log - Logger object for output (defaults to console if not provided) + * @returns {Promise} + * @todo Replace basic whitespace splitting with proper tokenizer (obvious-json) + */ async executeRawCommand(text, log) { // TODO: add obvious-json as a tokenizer const args = text.split(/\s+/); await this.executeCommand(args, log); } + + /** + * Gets a list of all registered command names/IDs + * @returns {string[]} Array of command identifier strings + */ get commandNames() { return this.commands_.map(command => command.id); } diff --git a/src/backend/src/services/CommentService.js b/src/backend/src/services/CommentService.js index 3420ccd30e..11dab61944 100644 --- a/src/backend/src/services/CommentService.js +++ b/src/backend/src/services/CommentService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const APIError = require("../api/APIError"); const FSNodeParam = require("../api/filesystem/FSNodeParam"); const { get_user } = require("../helpers"); @@ -6,15 +7,37 @@ const { Endpoint } = require("../util/expressutil"); const BaseService = require("./BaseService"); const { DB_WRITE } = require("./database/consts"); + +/** +* CommentService class handles all comment-related functionality in the system. +* Extends BaseService to provide comment creation, retrieval, and attachment capabilities +* for filesystem entries. Manages database operations for user comments and their +* associations with filesystem nodes. Provides REST API endpoints for comment +* operations including posting new comments and listing existing comments. +* @extends BaseService +*/ class CommentService extends BaseService { static MODULES = { uuidv4: require('uuid').v4, } + /** + * Static module dependencies used by the CommentService class + * @property {Function} uuidv4 - UUID v4 generator function from the uuid package + */ _init () { const svc_database = this.services.get('database'); this.db = svc_database.get(DB_WRITE, 'notification'); } ['__on_install.routes'] (_, { app }) { + /** + * Installs route handlers for comment-related endpoints + * Sets up POST routes for creating and listing comments on filesystem entries + * + * @param {*} _ Unused parameter + * @param {Object} options Installation options + * @param {Express} options.app Express application instance + * @private + */ const r_comment = (() => { const require = this.require; const express = require('express'); @@ -43,9 +66,6 @@ class CommentService extends BaseService { }); if ( req.body.version ) { - // this.attach_comment_to_fsentry_version({ - // node, comment, version, - // }); res.status(400).send('not implemented yet'); return; } else { @@ -81,9 +101,6 @@ class CommentService extends BaseService { }); if ( req.body.version ) { - // this.attach_comment_to_fsentry_version({ - // node, comment, version, - // }); res.status(400).send('not implemented yet'); return; } else { @@ -113,6 +130,16 @@ class CommentService extends BaseService { } + + /** + * Creates a new comment with the given text + * + * @param {Object} params - The parameters object + * @param {Object} params.req - Express request object containing user and body data + * @param {Object} params.res - Express response object + * @returns {Promise} The created comment object with id and uid + * @throws {APIError} If text field is missing from request body + */ async create_comment_ ({ req, res }) { if ( ! req.body.text ) { throw APIError.create('field_missing', null, { key: 'text' }); @@ -135,6 +162,15 @@ class CommentService extends BaseService { }; } + + /** + * Attaches a comment to a filesystem entry + * + * @param {Object} params - The parameters object + * @param {Object} params.node - The filesystem node to attach the comment to + * @param {Object} params.comment - The comment object containing id and other details + * @returns {Promise} Resolves when comment is successfully attached + */ async attach_comment_to_fsentry ({ node, comment }) { await this.db.write( 'INSERT INTO `user_fsentry_comments` ' + @@ -144,6 +180,14 @@ class CommentService extends BaseService { ); } + + /** + * Retrieves all comments associated with a filesystem entry + * + * @param {Object} params - The parameters object + * @param {Object} params.node - The filesystem node to get comments for + * @returns {Promise} Array of comment objects with user info attached + */ async get_comments_for_fsentry ({ node }) { const comments = await this.db.read( 'SELECT * FROM `user_comments` ' + diff --git a/src/backend/src/services/ConfigurableCountingService.js b/src/backend/src/services/ConfigurableCountingService.js index b3229699d6..b2ad471b06 100644 --- a/src/backend/src/services/ConfigurableCountingService.js +++ b/src/backend/src/services/ConfigurableCountingService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -22,11 +23,21 @@ const { Context } = require("../util/context"); const { DB_WRITE } = require('./database/consts'); const hash = v => { - var sum = crypto.createHash('sha1'); - sum.update('foo'); + const sum = crypto.createHash('sha1'); + sum.update(v); return sum.digest(); } + +/** +* @class ConfigurableCountingService +* @extends BaseService +* @description The ConfigurableCountingService class extends BaseService and is responsible for managing and incrementing +* configurable counting types for different services. It handles the initialization of the database connection, +* defines counting types and SQL columns, and provides a method to increment counts based on specific service +* types and values. This class is used to manage usage counts for various services, ensuring accurate tracking +* and updating of counts in the database. +*/ class ConfigurableCountingService extends BaseService { static counting_types = { gpt: { @@ -73,10 +84,34 @@ class ConfigurableCountingService extends BaseService { ], } + + /** + * Initializes the database connection for the ConfigurableCountingService. + * This method sets up the database service for writing counting data. + * + * @async + * @function _init + * @returns {Promise} A promise that resolves when the database connection is established. + * @memberof ConfigurableCountingService + */ async _init () { this.db = this.services.get('database').get(DB_WRITE, 'counting'); } + + /** + * Increments the count for a given service based on the provided parameters. + * This method builds an SQL query to update the count and other custom values + * in the database. It handles different SQL dialects (MySQL and SQLite) and + * ensures that the pricing category is correctly hashed and stored. + * + * @param {Object} params - The parameters for incrementing the count. + * @param {string} params.service_name - The name of the service. + * @param {string} params.service_type - The type of the service. + * @param {Object} params.values - The values to be incremented. + * @throws {Error} If the service type is unknown or if there are no more available columns. + * @returns {Promise} A promise that resolves when the count is successfully incremented. + */ async increment ({ service_name, service_type, values }) { values = values ? {...values} : {}; diff --git a/src/backend/src/services/Container.js b/src/backend/src/services/Container.js index 44878f8f3a..b7c9f224d2 100644 --- a/src/backend/src/services/Container.js +++ b/src/backend/src/services/Container.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -20,9 +21,18 @@ const { AdvancedBase } = require("@heyputer/putility"); const config = require("../config"); const { Context } = require("../util/context"); const { CompositeError } = require("../util/errorutil"); -const { TeePromise } = require("../util/promise"); +const { TeePromise } = require('@heyputer/putility').libs.promise; // 17 lines of code instead of an entire dependency-injection framework +/** +* The `Container` class is a lightweight dependency-injection container designed to manage +* service instances within the application. It provides functionality for registering, +* retrieving, and managing the lifecycle of services, including initialization and event +* handling. This class is intended to simplify dependency management and ensure that services +* are properly initialized and available throughout the application. +* +* @class +*/ class Container { constructor ({ logger }) { this.logger = logger; @@ -67,7 +77,10 @@ class Container { const my_config = config.services?.[name] || {}; const instance = cls.getInstance ? cls.getInstance({ services: this, config, my_config, name, args }) - : new cls({ services: this, config, my_config, name, args }) ; + : new cls({ + context: Context.get(), + services: this, config, my_config, name, args + }) ; this.instances_[name] = instance; if ( this.modname_ ) { @@ -122,6 +135,12 @@ class Container { } } has (name) { return !! this.instances_[name]; } + /** + * Checks if a service is registered in the container. + * + * @param {String} name - The name of the service to check. + * @returns {Boolean} - Returns true if the service is registered, false otherwise. + */ get values () { const values = {}; for ( const k in this.instances_ ) { @@ -141,6 +160,18 @@ class Container { return this.instances_; } + + /** + * Initializes all registered services in the container. + * + * This method first constructs each service by calling its `construct` method, + * and then initializes each service by calling its `init` method. If any service + * initialization fails, it logs the failures and throws a `CompositeError` + * containing details of all failed initializations. + * + * @returns {Promise} A promise that resolves when all services are + * initialized or rejects if any service initialization fails. + */ async init () { for ( const k in this.instances_ ) { this.logger.info(`constructing ${k}`); @@ -166,20 +197,40 @@ class Container { } } + + /** + * Emits an event to all registered services. + * + * This method sends an event identified by `id` along with any additional arguments to all + * services registered in the container. If a logger is available, it logs the event. + * + * @param {string} id - The identifier of the event. + * @param {...*} args - Additional arguments to pass to the event handler. + * @returns {Promise} A promise that resolves when all event handlers have completed. + */ async emit (id, ...args) { if ( this.logger ) { - this.logger.noticeme(`services:event ${id}`, { args }); + this.logger.info(`services:event ${id}`, { args }); } + const promises = []; for ( const k in this.instances_ ) { if ( this.instances_[k].__on ) { - promises.push(this.instances_[k].__on(id, args)); + promises.push(Context.arun(() => this.instances_[k].__on(id, args))); } } await Promise.all(promises); } } + +/** +* @class ProxyContainer +* @classdesc The ProxyContainer class is a proxy for the Container class, allowing for delegation of service management tasks. +* It extends the functionality of the Container class by providing a delegation mechanism. +* This class is useful for scenarios where you need to manage services through a proxy, +* enabling additional flexibility and control over service instances. +*/ class ProxyContainer { constructor (delegate) { this.delegate = delegate; @@ -200,6 +251,12 @@ class ProxyContainer { } return this.delegate.has(name); } + /** + * Checks if the container has a service with the specified name. + * + * @param {string} name - The name of the service to check. + * @returns {boolean} - Returns true if the service exists, false otherwise. + */ get values () { const values = {}; Object.assign(values, this.delegate.values); diff --git a/src/backend/src/services/ContextInitService.js b/src/backend/src/services/ContextInitService.js index 3267d8819a..66ce054509 100644 --- a/src/backend/src/services/ContextInitService.js +++ b/src/backend/src/services/ContextInitService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"claude"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -20,7 +21,24 @@ const { Context } = require("../util/context"); const BaseService = require("./BaseService"); // DRY: (2/3) - src/util/context.js; move install() to base class +/** +* @class ContextInitExpressMiddleware +* @description Express middleware that initializes context values for requests. +* Manages a collection of value initializers that can be synchronous values +* or asynchronous factory functions. Each initializer sets a key-value pair +* in the request context. Part of a DRY implementation shared with context.js. +* TODO: Consider moving install() method to base class. +*/ class ContextInitExpressMiddleware { + /** + * Express middleware class that initializes context values for requests + * + * Manages a list of value initializers that populate the Context with + * either static values or async-generated values when handling requests. + * Part of DRY pattern with src/util/context.js. + * + * @class + */ constructor () { this.value_initializers_ = []; } @@ -30,6 +48,11 @@ class ContextInitExpressMiddleware { install (app) { app.use(this.run.bind(this)); } + /** + * Installs the middleware into the Express application + * @param {Express} app - The Express application instance + * @returns {void} + */ async run (req, res, next) { const x = Context.get(); for ( const initializer of this.value_initializers_ ) { @@ -43,7 +66,23 @@ class ContextInitExpressMiddleware { } } + +/** +* @class ContextInitService +* @extends BaseService +* @description Service responsible for initializing and managing context values in the application. +* Provides methods to register both synchronous values and asynchronous factories for context +* initialization. Works in conjunction with Express middleware to ensure proper context setup +* for each request. Extends BaseService to integrate with the application's service architecture. +*/ class ContextInitService extends BaseService { + /** + * Service for initializing request context with values and async factories. + * Extends BaseService to provide middleware for Express that populates the Context + * with registered values and async-generated values at the start of each request. + * + * @extends BaseService + */ _construct () { this.mw = new ContextInitExpressMiddleware(); } @@ -57,6 +96,11 @@ class ContextInitService extends BaseService { key, async_factory, }); } + /** + * Registers an asynchronous factory function to initialize a context value + * @param {string} key - The key to store the value under in the context + * @param {Function} async_factory - Async function that returns the value to store + */ async ['__on_install.middlewares.context-aware'] (_, { app }) { this.mw.install(app); await this.services.emit('install.context-initializers'); diff --git a/src/backend/src/services/DBKVService.js b/src/backend/src/services/DBKVService.js new file mode 100644 index 0000000000..149b4bf4cc --- /dev/null +++ b/src/backend/src/services/DBKVService.js @@ -0,0 +1,221 @@ +const { get_app } = require("../helpers"); +const { Context } = require("../util/context"); +const BaseService = require("./BaseService"); +const { DB_READ } = require("./database/consts"); + +class DBKVService extends BaseService { + static MODULES = { + murmurhash: require('murmurhash'), + } + + _init () { + this.db = this.services.get('database').get(DB_READ, 'kvstore'); + } + + static IMPLEMENTS = { + ['puter-kvstore']: { + async get ({ app_uid, key }) { + const actor = Context.get('actor'); + + // If the actor is an app then it gets its own KV store. + // The way this is implemented isn't ideal for future behaviour; + // a KV implementation specified by the user would have parameters + // that are scoped to the app, so this should eventually be + // changed to get the app ID from the same interface that would + // be used to obtain per-app user-specified implementation params. + let app = actor.type?.app ?? undefined; + const user = actor.type?.user ?? undefined; + + if ( ! user ) throw new Error('User not found'); + + if ( ! app && app_uid ) { + app = await get_app({ uid: app_uid }); + } + + if ( Array.isArray(key) ) { + const keys = key; + const key_hashes = keys.map(key => this.modules.murmurhash.v3(key)); + const rows = app ? await this.db.read( + `SELECT kkey, value FROM kv WHERE user_id=? AND app=? AND kkey_hash IN (?)`, + [ user.id, app.uid, key_hashes ] + ) : await this.db.read( + `SELECT kkey, value FROM kv WHERE user_id=? AND (app IS NULL OR app = 'global') ` + + `AND kkey_hash IN (${key_hashes.map(() => '?').join(',')})`, + [ user.id, key_hashes ] + ); + + const kv = {}; + rows.forEach(row => { + row.value = this.db.case({ + mysql: () => row.value, + otherwise: () => JSON.parse(row.value ?? 'null'), + })(); + kv[row.kkey] = row.value; + }); + + return keys.map(key => kv[key]); + } + + const key_hash = this.modules.murmurhash.v3(key); + const kv = app ? await this.db.read( + `SELECT * FROM kv WHERE user_id=? AND app=? AND kkey_hash=? LIMIT 1`, + [ user.id, app.uid, key_hash ] + ) : await this.db.read( + `SELECT * FROM kv WHERE user_id=? AND (app IS NULL OR app = 'global') AND kkey_hash=? LIMIT 1`, + [ user.id, key_hash ] + ); + + if ( kv[0] ) kv[0].value = this.db.case({ + mysql: () => kv[0].value, + otherwise: () => JSON.parse(kv[0].value ?? 'null'), + })(); + + return kv[0]?.value ?? null; + }, + async set ({ app_uid, key, value }) { + const actor = Context.get('actor'); + const config = this.global_config; + + // Validate the key + // get() doesn't String() the key but it only passes it to + // murmurhash.v3() so it doesn't need to ¯\_(ツ)_/¯ + key = String(key); + if ( Buffer.byteLength(key, 'utf8') > config.kv_max_key_size ) { + throw new Error(`key is too large. Max size is ${config.kv_max_key_size}.`); + } + + // Validate the value + value = value === undefined ? null : value; + if ( + value !== null && + Buffer.byteLength(JSON.stringify(value), 'utf8') > + config.kv_max_value_size + ) { + throw new Error(`value is too large. Max size is ${config.kv_max_value_size}.`); + } + + let app = actor.type?.app ?? undefined; + const user = actor.type?.user ?? undefined; + if ( ! user ) throw new Error('User not found'); + + if ( ! app && app_uid ) { + app = await get_app({ uid: app_uid }); + } + + const key_hash = this.modules.murmurhash.v3(key); + + try { + await this.db.write( + `INSERT INTO kv (user_id, app, kkey_hash, kkey, value) + VALUES (?, ?, ?, ?, ?) ` + + this.db.case({ + mysql: 'ON DUPLICATE KEY UPDATE value = ?', + sqlite: 'ON CONFLICT(user_id, app, kkey_hash) DO UPDATE SET value = excluded.value', + }), + [ + user.id, app?.uid ?? 'global', key_hash, key, + JSON.stringify(value), + ...this.db.case({ mysql: [value], otherwise: [] }), + ] + ); + } catch (e) { + // I discovered that my .sqlite file was corrupted and the update + // above didn't work. The current database initialization does not + // cause this issue so I'm adding this log as a safeguard. + // - KernelDeimos / ED + const svc_error = this.services.get('error-service'); + svc_error.report('kvstore:sqlite_error', { + message: 'Broken database version - please contact maintainers', + source: e, + }); + } + + return true; + }, + async del ({ app_uid, key }) { + const actor = Context.get('actor'); + + let app = actor.type?.app ?? undefined; + const user = actor.type?.user ?? undefined; + if ( ! user ) throw new Error('User not found'); + + if ( ! app && app_uid ) { + app = await get_app({ uid: app_uid }); + } + + const key_hash = this.modules.murmurhash.v3(key); + + await this.db.write( + `DELETE FROM kv WHERE user_id=? AND app=? AND kkey_hash=?`, + [ user.id, app?.uid ?? 'global', key_hash ] + ); + + return true; + }, + async list ({ app_uid, as }) { + const actor = Context.get('actor'); + + let app = actor.type?.app ?? undefined; + const user = actor.type?.user ?? undefined; + + if ( ! app && app_uid ) { + app = await get_app({ uid: app_uid }); + } + + if ( ! user ) throw new Error('User not found'); + + let rows = app ? await this.db.read( + `SELECT kkey, value FROM kv WHERE user_id=? AND app=?`, + [ user.id, app.uid ] + ) : await this.db.read( + `SELECT kkey, value FROM kv WHERE user_id=? AND (app IS NULL OR app = 'global')`, + [ user.id ] + ); + + rows = rows.map(row => ({ + key: row.kkey, + value: this.db.case({ + mysql: () => row.value, + otherwise: () => JSON.parse(row.value ?? 'null') + })(), + })); + + as = as || 'entries'; + + if ( ! ['keys','values','entries'].includes(as) ) { + throw APIError.create('field_invalid', null, { + key: 'as', + expected: '"keys", "values", or "entries"', + }); + } + + if ( as === 'keys' ) rows = rows.map(row => row.key); + else if ( as === 'values' ) rows = rows.map(row => row.value); + + return rows; + }, + async flush ({ app_uid }) { + const actor = Context.get('actor'); + + let app = actor.type?.app ?? undefined; + const user = actor.type?.user ?? undefined; + if ( ! user ) throw new Error('User not found'); + + if ( ! app && app_uid ) { + app = await get_app({ uid: app_uid }); + } + + await this.db.write( + `DELETE FROM kv WHERE user_id=? AND app=?`, + [ user.id, app?.uid ?? 'global' ] + ); + + return true; + }, + } + }; +} + +module.exports = { + DBKVService, +}; diff --git a/src/backend/src/services/DetailProviderService.js b/src/backend/src/services/DetailProviderService.js index e03dba1dfa..fafcbcad55 100644 --- a/src/backend/src/services/DetailProviderService.js +++ b/src/backend/src/services/DetailProviderService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"openai-completion","model":"gpt-4o"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -23,7 +24,23 @@ const BaseService = require("./BaseService") * detail providers. A detail provider is a function that takes an * input object and uses its values to populate another object. */ +/** +* @class DetailProviderService +* @extends BaseService +* @description This class manages a collection of detail providers, +* which are functions that accept an input object to populate another object +* with relevant details. It provides methods to register new providers and +* retrieve details using all registered providers in sequence. +*/ class DetailProviderService extends BaseService { + /** + * Retrieves detailed information by invoking all registered detail providers with the given context. + * Each provider is expected to modify the out object directly. + * + * @param {Object} context - The input context for the providers. + * @param {Object} [out={}] - An object to store the combined results from each provider. + * @returns {Object} The combined results populated by the detail providers. + */ _construct () { this.providers_ = []; } @@ -32,6 +49,20 @@ class DetailProviderService extends BaseService { this.providers_.push(fn); } + + /** + * Asynchronously retrieves details by invoking registered detail providers + * in sequence. Populates the provided output object with the results of + * each provider. If no output object is provided, a new one is created + * by default. + * + * @param {Object} context - The context object containing input data for + * the providers. + * @param {Object} [out={}] - An optional output object to populate with + * the details. + * @returns {Promise} The populated output object after all + * providers have been processed. + */ async get_details (context, out) { out = out || {}; diff --git a/src/backend/src/services/DevConsoleService.js b/src/backend/src/services/DevConsoleService.js index 93910ac665..9eba6d744e 100644 --- a/src/backend/src/services/DevConsoleService.js +++ b/src/backend/src/services/DevConsoleService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"openai-completion","model":"gpt-4o-mini"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -19,11 +20,24 @@ const { consoleLogManager } = require('../util/consolelog'); const BaseService = require('./BaseService'); + +/** +* DevConsoleService - A service for managing the developer console interface, +* providing functionalities such as adding and removing widgets, updating the display, +* and handling command input/output in a terminal environment. +*/ class DevConsoleService extends BaseService { static MODULES = { fs: require('fs'), } + + /** + * Initializes the DevConsoleService instance, setting up required properties + * and determining if the application is running in a Docker environment. + * + * @returns {void} This method does not return a value. + */ _construct () { this.static_lines = []; this.widgets = []; @@ -42,7 +56,18 @@ class DevConsoleService extends BaseService { } } + + /** + * Activates the warning lights by adding a widget that outputs a warning message. + * The widget will display a flashing warning message when it is turned on. + */ turn_on_the_warning_lights () { + /** + * Turns on the warning lights by adding a warning widget to the console. + * This function calls the `add_widget` method to display a formatted + * warning message. The widget will be shown in red and blinking, + * indicating a warning state. + */ this.add_widget(() => { return `\x1B[31;1m\x1B[5m *** ${ Array(3).fill('WARNING').join(' ** ') @@ -66,6 +91,16 @@ class DevConsoleService extends BaseService { this.mark_updated(); } + + /** + * Updates the displayed output based on the current state of widgets. + * This method collects output from all active widgets, handles any errors, + * and maintains the integrity of displayed information. + * + * It modifies the static_lines array to reflect the latest outputs and + * removes widgets that produce errors. The display is updated only if there + * are changes in the output. + */ update_ () { const initialOutput = [...this.static_lines]; this.static_lines = []; @@ -86,6 +121,7 @@ class DevConsoleService extends BaseService { this.static_lines.push(...output); } + // The desired minimum output lines for display; used to ensure there's enough space for content. const DESIRED_MIN_OUT = 10; const size_ok = () => process.stdout.rows - DESIRED_MIN_OUT > this.static_lines.length; @@ -133,13 +169,28 @@ class DevConsoleService extends BaseService { return a.length === b.length && a.every((val, index) => val === b[index]); } + + /** + * Marks that an update has occurred in the DevConsoleService. + * This method sets the has_updates flag to true, indicating that + * the service should refresh the display during the next render cycle. + */ mark_updated () { this.has_updates = true; } + + /** + * Initializes the DevConsoleService, setting up necessary interfaces + * and handlers for command execution. This method manages input and + * output streams, integrates command completion, and refreshes console + * output based on widget updates. + * + * @async + * @returns {Promise} Resolves when the initialization completes. + */ async _init () { const services = this.services; - // await services.ready; const commands = services.get('commands'); const readline = require('readline'); @@ -171,9 +222,19 @@ class DevConsoleService extends BaseService { await commands.executeRawCommand(input, console); } this._after_cmd(); - // rl.prompt(); }); + + /** + * Handles the initialization of the DevConsole service, setting up + * command line interface and managing input/output operations. + * + * This method creates a readline interface for user input, processes + * commands, and manages the display of command output in the console. + * + * @async + * @returns {Promise} A promise that resolves when the initialization is complete. + */ this._before_cmd = () => { rl.pause(); rl.output.write('\x1b[1A\r'); @@ -185,6 +246,13 @@ class DevConsoleService extends BaseService { ); } + + /** + * _after_cmd - Handles operations needed after a command is executed. + * + * This method is called to clean up the output after a command has been processed. + * It logs a formatted message indicating the end of the command output. + */ this._after_cmd = () => { console.log( `\x1B[33m` + @@ -193,6 +261,12 @@ class DevConsoleService extends BaseService { ); } + + /** + * Prepares the output console by pausing input, clearing the console lines, + * and writing the static lines to be displayed. This method interacts directly + * with the terminal output handling the cursor movements and line clearances. + */ this._pre_write = () => { rl.pause(); process.stdout.write('\x1b[0m'); @@ -203,6 +277,13 @@ class DevConsoleService extends BaseService { } } + + /** + * Prepares the console for output by performing necessary actions + * such as clearing previous lines and setting up the environment + * for rendering new output. This method is called before new + * data is written to the console. + */ this._post_write = () => { this.update_(); // Draw separator bar @@ -233,11 +314,22 @@ class DevConsoleService extends BaseService { } }; + + /** + * Triggers the pre-write and post-write processes to refresh the console output. + * This method ensures that the latest changes are reflected in the console view. + */ this._redraw = () => { this._pre_write(); this._post_write(); }; + + /** + * Starts an interval that periodically checks for updates and redraws the console output if needed. + * The interval runs every 2000 milliseconds (2 seconds) and invokes the `_redraw` method if + * any updates have occurred since the last check. + */ setInterval(() => { if (this.has_updates) { this._redraw(); @@ -248,21 +340,18 @@ class DevConsoleService extends BaseService { consoleLogManager.decorate_all(({ replace }, ...args) => { this._pre_write(); }); + /** + * Decorates all console log messages with the specified pre-write actions. + * + * This method is invoked before each log message is printed to the console, + * ensuring that any necessary updates or redrawing of the console UI + * can occur before the message is displayed. + * + * It does not accept any parameters and does not return any value. + */ consoleLogManager.post_all(() => { this._post_write(); }) - // logService.loggers.unshift({ - // onLogMessage: () => { - // rl.pause(); - // rl.output.write('\x1b[2K\r'); - // } - // }); - // logService.loggers.push({ - // onLogMessage: () => { - // rl.resume(); - // rl._refreshLine(); - // } - // }); // This prevents the promptline background from staying // when Ctrl+C is used to terminate the server diff --git a/src/backend/src/services/DevTODService.js b/src/backend/src/services/DevTODService.js index 661cf1982c..7d34bcca15 100644 --- a/src/backend/src/services/DevTODService.js +++ b/src/backend/src/services/DevTODService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"claude"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -50,6 +51,13 @@ const tips = ( .filter((line) => line.length) ; + +/** +* Wraps text to specified width by breaking it into lines +* @param {string} text - The text to wrap +* @param {number} width - Maximum width of each line +* @returns {string[]} Array of wrapped text lines +*/ const wordwrap = (text, width) => { const lines = []; while ( text.length ) { @@ -59,11 +67,32 @@ const wordwrap = (text, width) => { return lines; }; + +/** +* @class DevTODService +* @extends BaseService +* @description Service that manages the "Tip of the Day" functionality in the development console. +* Displays random helpful tips about the system during startup and provides commands to manage +* the tip display. Inherits from BaseService and integrates with the dev-console and commands +* services to provide an interactive tip system for developers. +*/ class DevTODService extends BaseService { + /** + * DevTODService class - Manages "Tip of the Day" functionality for the developer console + * @extends BaseService + * @description Provides random development tips and console commands for managing tip display + * Integrates with the dev console to show helpful tips about source code and CLI usage + */ async _init () { const svc_commands = this.services.get('commands'); this._register_commands(svc_commands); } + /** + * Initializes the DevTODService by registering commands with the command service + * @private + * @async + * @returns {Promise} + */ async ['__on_boot.consolidation'] () { let random_tip = tips[Math.floor(Math.random() * tips.length)]; random_tip = wordwrap( @@ -71,12 +100,19 @@ class DevTODService extends BaseService { process.stdout.columns ? process.stdout.columns - 6 : 50 ); + /** + * Handles the boot consolidation phase for the Tip of the Day service + * Selects a random tip, wraps it to fit the console width, and creates + * a widget function to display the formatted tip with optional header/footer + * + * @returns {Promise} + */ this.tod_widget = () => { const lines = [ ...random_tip, ]; if ( ! this.global_config.minimal_console ) { - lines.unshift("\x1B[1mTip of the Day\x1B[0m"), + lines.unshift("\x1B[1mTip of the Day\x1B[0m"); lines.push("Type tod:dismiss to un-stick this message"); } surrounding_box('33;1', lines); diff --git a/src/backend/src/services/EmailService.js b/src/backend/src/services/EmailService.js index a11bd18ebb..b31fe63a98 100644 --- a/src/backend/src/services/EmailService.js +++ b/src/backend/src/services/EmailService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -162,6 +163,15 @@ If this was not you, please contact support@puter.com immediately. }, } + +/** +* @class EmailService +* @extends BaseService +* @description The EmailService class handles the sending of emails using predefined templates. +* It utilizes the nodemailer library for sending emails and Handlebars for template rendering. +* The class includes methods for constructing and initializing the service, getting the email transport, +* and sending emails with provided templates and values. +*/ class Emailservice extends BaseService { static MODULES = { nodemailer: require('nodemailer'), @@ -169,6 +179,16 @@ class Emailservice extends BaseService { dedent: require('dedent'), }; + + /** + * Initializes the EmailService by compiling email templates. + * + * This method compiles the email templates using Handlebars and dedent + * to ensure that they are ready for use. It stores the compiled templates + * in an object for quick access. + * + * @returns {void} + */ _construct () { this.templates = TEMPLATES; @@ -189,9 +209,26 @@ class Emailservice extends BaseService { } } + + /** + * Initializes the email service. + * This method is called during the initialization phase of the service. + * It sets up any necessary configurations or resources needed for the service to function correctly. + * + * @returns {void} + */ _init () { } + + /** + * Configures and initializes the email transport using Nodemailer. + * + * This method sets up the email transport configuration based on the provided settings and + * returns a configured Nodemailer transport object. + * + * @returns {Object} The configured Nodemailer transport object. + */ get_transport_ () { const nodemailer = this.modules.nodemailer; @@ -203,6 +240,18 @@ class Emailservice extends BaseService { return transport; } + + /** + * Sends an email using the configured transport and template. + * + * This method constructs an email message by applying the provided values to the specified template, + * then sends the email using the configured transport. + * + * @param {Object} user - The user object containing the email address. + * @param {string} template - The template key to use for constructing the email. + * @param {Object} values - The values to apply to the template. + * @returns {Promise} - A promise that resolves when the email is sent. + */ async send_email (user, template, values) { const email = user.email; diff --git a/src/backend/src/services/EngPortalService.js b/src/backend/src/services/EngPortalService.js index ce507568ef..54859429af 100644 --- a/src/backend/src/services/EngPortalService.js +++ b/src/backend/src/services/EngPortalService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -18,9 +19,19 @@ */ const { AdvancedBase } = require("@heyputer/putility"); + +/** +* @class EngPortalService +* @extends {AdvancedBase} +* +* EngPortalService is a class that provides services for managing and accessing various operations, alarms, and statistics +* within a system. It inherits from the AdvancedBase class and utilizes multiple dependencies such as socket.io for communication +* and uuidv4 for generating unique identifiers. The class includes methods for listing operations, serializing frames, listing alarms, +* fetching server statistics, and registering command handlers. This class is integral to maintaining and monitoring system health +* and operations efficiently. +*/ class EngPortalService extends AdvancedBase { static MODULES = { - socketio: require('../socketio.js'), uuidv4: require('uuid').v4, }; @@ -31,6 +42,15 @@ class EngPortalService extends AdvancedBase { this._registerCommands(this.commands); } + + /** + * Lists all ongoing operations. + * This method retrieves all ongoing operations from the 'operationTrace' service, + * serializes them, and returns the serialized list. + * + * @async + * @returns {Promise} A list of serialized operation frames. + */ async list_operations () { const svc_operationTrace = this.services.get('operationTrace'); const ls = []; @@ -68,6 +88,14 @@ class EngPortalService extends AdvancedBase { return out; } + + /** + * Retrieves a list of alarms. + * + * This method fetches all active alarms from the 'alarm' service and returns a serialized array of alarm objects. + * + * @returns {Promise} A promise that resolves to an array of serialized alarm objects. + */ async list_alarms () { const svc_alarm = this.services.get('alarm'); const ls = []; @@ -79,6 +107,15 @@ class EngPortalService extends AdvancedBase { return ls; } + + /** + * Gets the system statistics. + * + * This method retrieves the system statistics from the server-health service and returns them. + * + * @async + * @returns {Promise} A promise that resolves to the system statistics. + */ async get_stats () { const svc_health = this.services.get('server-health'); return await svc_health.get_stats(); diff --git a/src/backend/src/services/EntityStoreService.js b/src/backend/src/services/EntityStoreService.js index af7327783e..09ce7431fd 100644 --- a/src/backend/src/services/EntityStoreService.js +++ b/src/backend/src/services/EntityStoreService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"xai"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -17,12 +18,28 @@ * along with this program. If not, see . */ const APIError = require("../api/APIError"); +const { Entity } = require("../om/entitystorage/Entity"); const { IdentifierUtil } = require("../om/IdentifierUtil"); -const { Null } = require("../om/query/query"); +const { Null, And, Eq } = require("../om/query/query"); const { Context } = require("../util/context"); const BaseService = require("./BaseService"); +/** +* EntityStoreService - A service class that manages entity-related operations in the backend of Puter. +* This class extends BaseService to provide methods for creating, reading, updating, selecting, +* upserting, and deleting entities. It interacts with an upstream data provider to perform these +* operations, ensuring consistency and providing context-aware functionality for entity management. +*/ class EntityStoreService extends BaseService { + /** + * Initializes the EntityStoreService with necessary entity and upstream configurations. + * + * @param {Object} args - The initialization arguments. + * @param {string} args.entity - The name of the entity to operate on. Required. + * @param {Object} args.upstream - The upstream service to handle operations. + * + * @throws {Error} If `args.entity` is not provided. + */ async _init (args) { if ( ! args.entity ) { throw new Error('EntityStoreService requires an entity name'); @@ -30,6 +47,21 @@ class EntityStoreService extends BaseService { this.upstream = args.upstream; + + /** + * Initializes the EntityStoreService with the provided arguments. + * + * @param {Object} args - Initialization arguments. + * @param {string} args.entity - The name of the entity this service will manage. + * If not provided, an error is thrown. + * @param {Object} args.upstream - The upstream service or data source. + * + * @throws {Error} If the entity name is not provided in the arguments. + * + * @returns {Promise} A promise that resolves when initialization is complete. + * + * @note This method sets up the context for the entity operations and provides it to the upstream service. + */ const context = Context.get().sub({ services: this.services }); const om = this.services.get('registry').get('om:mapping').get(args.entity); this.om = om; @@ -39,14 +71,99 @@ class EntityStoreService extends BaseService { entity_name: args.entity, }); } + + static IMPLEMENTS = { + ['crud-q']: { + async create ({ object, options }) { + if ( object.hasOwnProperty(this.om.primary_identifier) ) { + throw APIError.create('field_not_allowed_for_create', null, { + key: this.om.primary_identifier + }); + } + const entity = await Entity.create({ om: this.om }, object); + return await this.create(entity, options); + }, + async update ({ object, id, options }) { + const entity = await Entity.create({ om: this.om }, object); + return await this.update(entity, id, options); + }, + async upsert ({ object, id, options }) { + const entity = await Entity.create({ om: this.om }, object); + return await this.upsert(entity, id, options); + }, + async read ({ uid, id }) { + if ( ! uid && ! id ) { + throw APIError.create('xor_field_missing', null, { + names: ['uid', 'id'], + }); + } + + const entity = await this.fetch_based_on_either_id_(uid, id); + if ( ! entity ) { + throw APIError.create('entity_not_found', null, { + identifier: uid + }); + } + return await entity.get_client_safe(); + }, + async select (options) { + const entities = await this.select(options); + const client_safe_entities = []; + for ( const entity of entities ) { + client_safe_entities.push(await entity.get_client_safe()); + } + return client_safe_entities; + }, + async delete ({ uid, id }) { + if ( ! uid && ! id ) { + throw APIError.create('xor_field_missing', null, { + names: ['uid', 'id'], + }); + } + + if ( id && ! uid ) { + const entity = await this.fetch_based_on_complex_id_(id); + if ( ! entity ) { + throw APIError.create('entity_not_found', null, { + identifier: id + }); + } + uid = await entity.get(this.om.primary_identifier); + } + + return await this.delete(uid); + } + } + }; // TODO: can replace these with MethodProxyFeature + /** + * Retrieves an entity by its unique identifier. + * + * @param {string} uid - The unique identifier of the entity to read. + * @returns {Promise} The entity object if found, otherwise null or throws an error. + * @throws {APIError} If the entity with the given uid does not exist. + */ async create (entity, options) { return await this.upstream.upsert(entity, { old_entity: null, options }); } + /** + * Reads an entity from the upstream data store using its unique identifier. + * + * @param {string} uid - The unique identifier of the entity to read. + * @returns {Promise} A promise that resolves to the entity object if found. + * @throws {APIError} If the entity with the given `uid` does not exist. + */ async read (uid) { return await this.upstream.read(uid); } + /** + * Retrieves an entity by its unique identifier (UID). + * + * @param {string} uid - The unique identifier of the entity to retrieve. + * @returns {Promise} The entity associated with the given UID. + * @throws {Error} If the entity cannot be found or an error occurs during retrieval. + */ async select ({ predicate, ...rest }) { if ( ! predicate ) predicate = []; if ( Array.isArray(predicate) ) { @@ -56,6 +173,19 @@ class EntityStoreService extends BaseService { if ( ! predicate) predicate = new Null(); return await this.upstream.select({ predicate, ...rest }); } + /** + * Retrieves entities matching a given predicate. + * + * This method performs a selection query on the upstream data source. + * If no predicate is provided, it defaults to selecting all entities. + * + * @param {Object} options - The selection options. + * @param {Array|Function} options.predicate - The predicate for filtering entities. + * If an array, it's expected to be in the format [operator, ...args]. + * If not provided, it defaults to a Null predicate, effectively selecting all entities. + * @param {Object} [options.rest] - Additional options for the selection query. + * @returns {Promise} A promise that resolves to an array of entities matching the predicate. + */ async update (entity, id, options) { let old_entity = await this.read( await entity.get(this.om.primary_identifier)); @@ -86,6 +216,21 @@ class EntityStoreService extends BaseService { return await this.upstream.upsert(entity, { old_entity, options }); } + /** + * Updates an existing entity in the store. + * + * @param {Object} entity - The entity to update with new values. + * @param {string|number} id - The identifier of the entity to update. Can be a string or number. + * @param {Object} options - Additional options for the update operation. + * @returns {Promise} The updated entity after the operation. + * @throws {APIError} If the entity to be updated is not found. + * + * @note This method first attempts to fetch the entity by its primary identifier. If not found, + * it uses `IdentifierUtil` to detect and fetch by other identifiers if provided. + * If the entity still isn't found, an error is thrown. The method ensures that the + * entity's primary identifier is updated to match the existing entity before performing + * the actual update through `this.upstream.upsert`. + */ async upsert (entity, id, options) { let old_entity = await this.read( await entity.get(this.om.primary_identifier)); @@ -112,6 +257,18 @@ class EntityStoreService extends BaseService { return await this.upstream.upsert(entity, { old_entity, options }); } + /** + * Deletes an entity from the store. + * + * @param {string} uid - The unique identifier of the entity to delete. + * @returns {Promise} A promise that resolves when the entity is deleted. + * @throws {APIError} If the entity with the given `uid` is not found. + * + * This method first attempts to read the entity with the given `uid`. If the entity + * does not exist, it throws an `APIError` with the message 'entity_not_found'. + * If the entity exists, it calls the upstream service to delete the entity, + * passing along the old entity data for reference. + */ async delete (uid) { const old_entity = await this.read(uid); if ( ! old_entity ) { @@ -121,6 +278,68 @@ class EntityStoreService extends BaseService { } return await this.upstream.delete(uid, { old_entity }); } + + async fetch_based_on_complex_id_ (id) { + // Ensure `id` is an object and get its keys + if ( ! id || typeof id !== 'object' || Array.isArray(id) ) { + throw APIError.create('invalid_id', null, { id }); + } + + const id_keys = Object.keys(id); + // sort keys alphabetically + id_keys.sort(); + + // Ensure key set is valid based on redundant keys listing + const redundant_identifiers = this.om.redundant_identifiers ?? []; + + let match_found = false; + for ( let key of redundant_identifiers ) { + // Either a single key or a list + key = Array.isArray(key) ? key : [key]; + + // All keys in the list must be present in the id + for ( let i=0 ; i < key.length ; i++ ) { + if ( ! id_keys.includes(key[i]) ) { + break; + } + if ( i === key.length - 1 ) { + match_found = true; + break; + } + } + } + + if ( ! match_found ) { + throw APIError.create('invalid_id', null, { id }); + } + + // Construct a query predicate based on the keys + const key_eqs = []; + for ( const key of id_keys ) { + key_eqs.push(new Eq({ + key, + value: id[key], + })); + } + let predicate = new And({ children: key_eqs }); + + // Perform a select + const entity = await this.read({ predicate }); + if ( ! entity ) { + return null; + } + + // Ensure there is only one result + return entity; + } + + async fetch_based_on_either_id_ (uid, id) { + if ( uid ) { + return await this.read(uid); + } + + return await this.fetch_based_on_complex_id_(id); + } } module.exports = { diff --git a/src/backend/src/services/EventService.js b/src/backend/src/services/EventService.js index 89417ed547..422230f10e 100644 --- a/src/backend/src/services/EventService.js +++ b/src/backend/src/services/EventService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"openai-completion","model":"gpt-4o-mini"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -19,6 +20,13 @@ const { Context } = require("../util/context"); const BaseService = require("./BaseService"); + +/** +* EventService class extends BaseService to provide a mechanism for +* emitting and listening to events within the application. It manages +* event listeners scoped to specific keys and allows global listeners +* for broader event handling. +*/ class ScopedEventBus { constructor (event_bus, scope) { this.event_bus = event_bus; @@ -34,7 +42,22 @@ class ScopedEventBus { } } + +/** +* Class representing the EventService, which extends the BaseService. +* This service is responsible for managing event listeners and emitting +* events within a scoped context, allowing for flexible event handling +* and decoupled communication between different parts of the application. +*/ class EventService extends BaseService { + /** + * Initializes listeners and global listeners for the EventService. + * This method is called to set up the internal data structures needed + * for managing event listeners upon construction of the service. + * + * @async + * @returns {Promise} A promise that resolves when the initialization is complete. + */ async _construct () { this.listeners_ = {}; this.global_listeners_ = []; @@ -51,9 +74,7 @@ class EventService extends BaseService { // actual emit const listeners = this.listeners_[part]; if ( ! listeners ) continue; - for ( let i = 0; i < listeners.length; i++ ) { - const callback = listeners[i]; - + for ( const callback of listeners ) { // IIAFE wrapper to catch errors without blocking // event dispatch. Context.arun(async () => { @@ -73,6 +94,16 @@ class EventService extends BaseService { for ( const callback of this.global_listeners_ ) { // IIAFE wrapper to catch errors without blocking // event dispatch. + /** + * Invokes all registered global listeners for an event with the provided key, data, and meta + * information. Each callback is executed within a context that handles errors gracefully, + * ensuring that one failing listener does not disrupt subsequent invocations. + * + * @param {string} key - The event key to emit. + * @param {*} data - The data to be passed to the listeners. + * @param {Object} [meta={}] - Optional metadata related to the event. + * @returns {void} + */ Context.arun(async () => { try { await callback(key, data, meta); @@ -95,6 +126,17 @@ class EventService extends BaseService { listeners.push(callback); const det = { + /** + * Registers a callback function for the specified event selector. + * + * This method will push the provided callback onto the list of listeners + * for the event specified by the selector. It returns an object containing + * a detach method, which can be used to remove the listener. + * + * @param {string} selector - The event selector to listen for. + * @param {Function} callback - The function to be invoked when the event is emitted. + * @returns {Object} An object with a detach method to unsubscribe the listener. + */ detach: () => { const idx = listeners.indexOf(callback); if ( idx !== -1 ) { diff --git a/src/backend/src/services/FeatureFlagService.js b/src/backend/src/services/FeatureFlagService.js index 65337fabad..7b9a797f50 100644 --- a/src/backend/src/services/FeatureFlagService.js +++ b/src/backend/src/services/FeatureFlagService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const { Context } = require("../util/context"); const { whatis } = require("../util/langutil"); const { PermissionUtil } = require("./auth/PermissionService"); @@ -7,13 +8,31 @@ const BaseService = require("./BaseService"); * FeatureFlagService is a way to let the client (frontend) know what features * are enabled or disabled for the current user. */ +/** +* @class FeatureFlagService +* @extends BaseService +* @description A service that manages feature flags to control feature availability across the application. +* Provides methods to register, check, and retrieve feature flags based on user permissions and configurations. +* Integrates with the permission system to determine feature access for different users. +* Supports both static configuration flags and dynamic function-based feature flags. +*/ class FeatureFlagService extends BaseService { + /** + * Initializes the FeatureFlagService instance by setting up an empty Map for known flags + * @private + * @method + */ _construct () { this.known_flags = new Map(); } register (name, spec) { this.known_flags.set(name, spec); } + /** + * Registers a new feature flag with the service + * @param {string} name - The name/identifier of the feature flag + * @param {Object|boolean} spec - The specification for the flag. Can be a boolean value or an object with $ property indicating flag type + */ async _init () { const svc_detailProvider = this.services.get('whoami'); svc_detailProvider.register_provider(async (context, out) => { @@ -21,9 +40,25 @@ class FeatureFlagService extends BaseService { out.feature_flags = await this.get_summary(context.actor); }); } + /** + * Initializes the feature flag service by registering a provider with the whoami service. + * This provider adds feature flag information to user details when requested. + * + * @async + * @private + * @returns {Promise} + */ async check (...a) { // allows binding call with multiple options objects; // the last argument is the permission to check + /** + * Checks if a feature flag is enabled for the given context + * @param {...Object} options - Configuration options objects that will be merged + * @param {string} permission - The feature flag name to check + * @returns {Promise} Whether the feature flag is enabled + * @description Processes multiple option objects and a permission string to determine + * if a feature flag is enabled. Handles config flags, function flags, and permission-based flags. + */ const { options, value: permission } = (() => { let value; const options = {}; @@ -63,6 +98,15 @@ class FeatureFlagService extends BaseService { return true; } + + /** + * Gets a summary of all feature flags for a given actor + * @param {Object} actor - The actor to check feature flags for + * @returns {Promise} Object mapping feature flag names to their values: + * - For config flags: returns the configured value + * - For function flags: returns result of calling the flag function + * - For permission flags: returns true if actor has any matching permissions, false otherwise + */ async get_summary (actor) { const summary = {}; for ( const [key, value] of this.known_flags.entries() ) { diff --git a/src/backend/src/services/FilesystemAPIService.js b/src/backend/src/services/FilesystemAPIService.js index b8872e5271..703df36cce 100644 --- a/src/backend/src/services/FilesystemAPIService.js +++ b/src/backend/src/services/FilesystemAPIService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"openai-completion","model":"gpt-4o-mini"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -18,7 +19,27 @@ */ const BaseService = require("./BaseService"); + +/** +* @class FilesystemAPIService +* @extends BaseService +* @description This service handles all filesystem-related API routes, +* allowing for operations like file creation, deletion, +* reading, and searching through a structured set of +* endpoints. It integrates with the web server to expose +* these functionalities for client use. +*/ class FilesystemAPIService extends BaseService { + /** + * Sets up the route handlers for the Filesystem API. + * This method registers various endpoints related to filesystem operations + * such as creating, deleting, reading, and updating files. It uses the + * web server's app instance to attach the corresponding routers. + * + * @async + * @function __on_install.routes + * @returns {Promise} A promise that resolves when the routes are set up. + */ async ['__on_install.routes'] () { const { app } = this.services.get('web-server'); diff --git a/src/backend/src/services/GetUserService.js b/src/backend/src/services/GetUserService.js index 9c3d2510e0..14ba320716 100644 --- a/src/backend/src/services/GetUserService.js +++ b/src/backend/src/services/GetUserService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"openai-completion","model":"gpt-4o"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -31,7 +32,17 @@ const { DB_READ } = require("./database/consts"); * * The original `get_user` function now uses this service. */ +/** +* Class representing a service to retrieve user information by various identifying properties. +* The GetUserService provides an interface for accessing user data while facilitating caching +* mechanisms to optimize database interactions, allowing for read operations against different +* identifiers such as username, email, UUID, and referral codes. +*/ class GetUserService extends BaseService { + /** + * Constructor for GetUserService. + * Initializes the set of identifying properties used to retrieve user data. + */ _construct () { this.id_properties = new Set(); @@ -41,8 +52,19 @@ class GetUserService extends BaseService { this.id_properties.add('email'); this.id_properties.add('referral_code'); } + /** + * Initializes the GetUserService instance, setting up the + * identifying properties used for user retrieval. + */ async _init () { } + /** + * Initializes the GetUserService instance. + * This method prepares any necessary internal structures or states. + * It is called automatically upon instantiation of the service. + * + * @returns {Promise} A promise that resolves when the initialization is complete. + */ async get_user (options) { const user = await this.get_user_(options); if ( ! user ) return null; @@ -51,6 +73,19 @@ class GetUserService extends BaseService { await svc_whoami.get_details({ user }, user); return user; } + /** + * Retrieves a user object based on the provided options. + * + * This method queries the user from cache or database, + * depending on the caching options provided. If the user + * is found, it also calls the 'whoami' service to enrich + * the user details before returning. + * + * @param {Object} options - The options for retrieving the user. + * @param {boolean} [options.cached=true] - Indicates if caching should be used. + * @param {boolean} [options.force=false] - Forces a read from the database regardless of cache. + * @returns {Promise} The user object if found, else null. + */ async get_user_ (options) { const services = this.services; @@ -96,11 +131,6 @@ class GetUserService extends BaseService { kv.set(`users:${prop}:${user[prop]}`, user); } } - // kv.set('users:username:' + user.username, user); - // kv.set('users:email:' + user.email, user); - // kv.set('users:uuid:' + user.uuid, user); - // kv.set('users:id:' + user.id, user); - // kv.set('users:referral_code:' + user.referral_code, user); } catch (e) { console.error(e); } diff --git a/src/backend/src/services/HelloWorldService.js b/src/backend/src/services/HelloWorldService.js index aa233ca0a7..9087e567c5 100644 --- a/src/backend/src/services/HelloWorldService.js +++ b/src/backend/src/services/HelloWorldService.js @@ -1,13 +1,34 @@ +// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}} const BaseService = require("./BaseService"); + +/** +* @class HelloWorldService +* @extends BaseService +* @description This class extends the BaseService and provides methods to get the version +* of the service and to generate a greeting message. The greeting message can be personalized +* based on the input subject. +*/ class HelloWorldService extends BaseService { static IMPLEMENTS = { ['version']: { + /** + * Returns the current version of the service. + * + * @returns {string} The version string. + */ get_version () { return 'v1.0.0'; } }, ['hello-world']: { + /** + * Greets the user with a customizable message. + * + * @param {Object} options - The options object. + * @param {string} [options.subject] - The subject of the greeting. If not provided, defaults to "World". + * @returns {string} The greeting message. + */ async greet ({ subject }) { if ( subject ) { return `Hello, ${subject}!`; diff --git a/src/backend/src/services/HostDiskUsageService.js b/src/backend/src/services/HostDiskUsageService.js index fa2c50e88d..a564efa016 100644 --- a/src/backend/src/services/HostDiskUsageService.js +++ b/src/backend/src/services/HostDiskUsageService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"xai"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -20,6 +21,16 @@ const { BaseService } = require("../../exports"); const { execSync } = require('child_process'); const config = require("../config"); + +/** +* The HostDiskUsageService class extends BaseService to provide functionality for monitoring +* and reporting disk usage on the host system. This service identifies the mount point or drive +* where the current process is running, and performs disk usage checks for that specific location. +* It supports different operating systems like macOS and Linux, with placeholders for future +* Windows support. +* +* @extends BaseService +*/ class HostDiskUsageService extends BaseService { static DESCRIPTION = ` This service is responsible for identifying the mountpoint/drive @@ -27,6 +38,18 @@ class HostDiskUsageService extends BaseService { disk usage of that mountpoint/drive. `; + + /** + * Initializes the service by determining the disk usage of the mountpoint/drive + * where the current working directory resides. + * + * @async + * @function + * @memberof HostDiskUsageService + * @instance + * @returns {Promise} A promise that resolves when initialization is complete. + * @throws {Error} If unable to determine disk usage for the platform. + */ async _init() { const current_platform = process.platform; @@ -49,6 +72,17 @@ class HostDiskUsageService extends BaseService { } // TODO: TTL cache this value + /** + * Retrieves the current disk usage for the host system. + * + * This method checks the disk usage of the mountpoint or drive + * where the current process is running, based on the operating system. + * + * @returns {number} The amount of disk space used in bytes. + * + * @note This method does not cache its results and should be optimized + * with a TTL cache to prevent excessive system calls. + */ get_host_usage () { const current_platform = process.platform; @@ -67,6 +101,13 @@ class HostDiskUsageService extends BaseService { } // Called by the /df endpoint + /** + * Retrieves extra disk usage information for the host. + * This method is used by the /df endpoint to gather + * additional statistics on host disk usage. + * + * @returns {Object} An object containing the host's disk usage data. + */ get_extra () { return { host_used: this.get_host_usage(), diff --git a/src/backend/src/services/KernelInfoService.js b/src/backend/src/services/KernelInfoService.js index e8807f2177..0e7aeb5508 100644 --- a/src/backend/src/services/KernelInfoService.js +++ b/src/backend/src/services/KernelInfoService.js @@ -1,18 +1,42 @@ +// METADATA // {"ai-commented":{"service":"claude"}} const configurable_auth = require("../middleware/configurable_auth"); const { Context } = require("../util/context"); const { Endpoint } = require("../util/expressutil"); const BaseService = require("./BaseService"); const { Interface } = require("./drivers/meta/Construct"); +// Permission flag that grants access to view all services in the kernel info system const PERM_SEE_ALL = 'kernel-info:see-all-services'; +// Permission flag that grants access to view all services in the kernel info system const PERM_SEE_DRIVERS = 'kernel-info:see-all-drivers'; + +/** +* KernelInfoService class provides information about the kernel's services, modules, and interfaces. +* It handles listing available modules, services, and their implementations based on user permissions. +* The service exposes endpoints for querying kernel module information and manages access control +* through permission checks for viewing all services and drivers. +* @extends BaseService +*/ class KernelInfoService extends BaseService { + /** + * Service for providing kernel and service information + * Extends BaseService to provide system-level information about services, interfaces and drivers + * Handles permissions and access control for viewing service information + * Exposes endpoints for listing modules and service information + */ async _init () { // } ['__on_install.routes'] (_, { app }) { + /** + * Installs routes for the kernel info service + * @param {*} _ Unused parameter + * @param {Object} param1 Object containing Express app instance + * @param {Express} param1.app Express application instance + * @private + */ const router = (() => { const require = this.require; const express = require('express'); @@ -54,15 +78,12 @@ class KernelInfoService extends BaseService { } const services = []; - const modules = []; for ( const k in this.services.modules_ ) { const module_info = { name: k, services: [] }; - modules.push(module_info); - for ( const s_k of this.services.modules_[k].services_l ) { const service_info = { name: s_k, diff --git a/src/backend/src/services/LocalDiskStorageService.js b/src/backend/src/services/LocalDiskStorageService.js index 5b9213838c..a9d01080ed 100644 --- a/src/backend/src/services/LocalDiskStorageService.js +++ b/src/backend/src/services/LocalDiskStorageService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -17,16 +18,34 @@ * along with this program. If not, see . */ const { LocalDiskStorageStrategy } = require("../filesystem/strategies/storage_a/LocalDiskStorageStrategy"); -const { TeePromise } = require("../util/promise"); +const { TeePromise } = require('@heyputer/putility').libs.promise; const { progress_stream, size_limit_stream } = require("../util/streamutil"); const BaseService = require("./BaseService"); + +/** +* @class LocalDiskStorageService +* @extends BaseService +* +* The LocalDiskStorageService class is responsible for managing local disk storage. +* It provides methods for storing, retrieving, and managing files on the local disk. +* This service extends the BaseService class to inherit common service functionalities. +*/ class LocalDiskStorageService extends BaseService { static MODULES = { fs: require('fs'), path: require('path'), } + + /** + * Initializes the context for the storage service. + * + * This method registers the LocalDiskStorageStrategy with the context + * initialization service and sets the storage for the mountpoint service. + * + * @returns {Promise} A promise that resolves when the context is initialized. + */ async ['__on_install.context-initializers'] () { const svc_contextInit = this.services.get('context-init'); const storage = new LocalDiskStorageStrategy({ services: this.services }); @@ -36,6 +55,14 @@ class LocalDiskStorageService extends BaseService { svc_mountpoint.set_storage(storage); } + + /** + * Initializes the local disk storage service. + * + * This method sets up the storage directory and ensures it exists. + * + * @returns {Promise} A promise that resolves when the initialization is complete. + */ async _init () { const require = this.require; const path_ = require('path'); @@ -53,6 +80,22 @@ class LocalDiskStorageService extends BaseService { return path.join(this.path, key); } + + /** + * Stores a stream to local disk storage. + * + * This method takes a stream and stores it on the local disk under the specified key. + * It also supports progress tracking and size limiting. + * + * @async + * @function store_stream + * @param {Object} options - The options object. + * @param {string} options.key - The key under which the stream will be stored. + * @param {number} options.size - The size of the stream. + * @param {stream.Readable} options.stream - The readable stream to be stored. + * @param {Function} [options.on_progress] - The callback function to track progress. + * @returns {Promise} A promise that resolves when the stream is fully stored. + */ async store_stream ({ key, size, stream, on_progress }) { const require = this.require; const fs = require('fs'); @@ -78,6 +121,17 @@ class LocalDiskStorageService extends BaseService { return await writePromise; } + + /** + * Stores a buffer to the local disk. + * + * This method writes a given buffer to a file on the local disk, identified by a key. + * + * @param {Object} params - The parameters object. + * @param {string} params.key - The key used to identify the file. + * @param {Buffer} params.buffer - The buffer containing the data to be stored. + * @returns {Promise} A promise that resolves when the buffer is successfully stored. + */ async store_buffer ({ key, buffer }) { const require = this.require; const fs = require('fs'); @@ -86,6 +140,14 @@ class LocalDiskStorageService extends BaseService { await fs.promises.writeFile(path, buffer); } + + /** + * Creates a read stream for a given key. + * + * @param {Object} options - The options object. + * @param {string} options.key - The key for which to create the read stream. + * @returns {stream.Readable} The read stream for the given key. + */ async create_read_stream ({ key }) { const require = this.require; const fs = require('fs'); @@ -94,6 +156,15 @@ class LocalDiskStorageService extends BaseService { return fs.createReadStream(path); } + + /** + * Copies a file from one key to another within the local disk storage. + * + * @param {Object} params - The parameters for the copy operation. + * @param {string} params.src_key - The source key of the file to be copied. + * @param {string} params.dst_key - The destination key where the file will be copied. + * @returns {Promise} A promise that resolves when the file is successfully copied. + */ async copy ({ src_key, dst_key }) { const require = this.require; const fs = require('fs'); @@ -104,6 +175,16 @@ class LocalDiskStorageService extends BaseService { await fs.promises.copyFile(src_path, dst_path); } + + /** + * Deletes a file from the local disk storage. + * + * This method removes the file associated with the given key from the storage. + * + * @param {Object} params - The parameters for the delete operation. + * @param {string} params.key - The key of the file to be deleted. + * @returns {Promise} - A promise that resolves when the file is successfully deleted. + */ async delete ({ key }) { const require = this.require; const fs = require('fs'); diff --git a/src/backend/src/services/LockService.js b/src/backend/src/services/LockService.js index f9eb78a6c9..2a3c3fce8c 100644 --- a/src/backend/src/services/LockService.js +++ b/src/backend/src/services/LockService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"openai-completion","model":"gpt-4o"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -25,10 +26,31 @@ const BaseService = require("./BaseService"); * * This serivces uses RWLock but always locks in write mode. */ +/** +* Represents the LockService class responsible for managing locks +* using reader-writer locks (RWLock). This service ensures that +* critical sections are properly handled by enforcing write locks +* exclusively, enabling safe concurrent access to shared resources +* while preventing race conditions and ensuring data integrity. +*/ class LockService extends BaseService { + /** + * Initializes the LockService by setting up the locks object + * and registering the 'lock' commands. This method is called + * during the service initialization phase. + */ async _construct () { this.locks = {}; } + /** + * Initializes the locks object to store lock instances. + * + * This method is called during the construction of the LockService + * instance to ensure that the locks property is ready for use. + * + * @returns {Promise} A promise that resolves when the + * initialization is complete. + */ async _init () { const svc_commands = this.services.get('commands'); svc_commands.registerCommands('lock', [ @@ -66,6 +88,17 @@ class LockService extends BaseService { ]); } + + /** + * Acquires a lock for the specified name, allowing for a callback to be executed while the lock is held. + * If the name is an array, all locks will be acquired in sequence. The method supports optional + * configurations, including a timeout feature. It returns the result of the callback execution. + * + * @param {string|string[]} name - The name(s) of the lock(s) to acquire. + * @param {Object} [opt_options] - Optional configuration options. + * @param {function} callback - The function to call while the lock is held. + * @returns {Promise} The result of the callback. + */ async lock (name, opt_options, callback) { if ( typeof opt_options === 'function' ) { callback = opt_options; @@ -78,6 +111,17 @@ class LockService extends BaseService { // TODO: verbose log option by service // console.log('LOCKING NAMES', names) const section = names.reduce((current_callback, name) => { + /** + * Acquires a lock for the specified name or names. + * + * If the name is an array, it locks each specified name in sequence. + * The method ensures that all specified locks are acquired before executing the callback. + * + * @param {string|string[]} name - The name(s) of the lock(s) to acquire. + * @param {Object} [opt_options={}] - Optional configuration for the lock operation. + * @param {Function} callback - Function to be executed once the lock is acquired. + * @returns {Promise} Resolves when the callback execution is complete. + */ return async () => { return await this.lock(name, opt_options, current_callback); }; @@ -98,6 +142,17 @@ class LockService extends BaseService { let timeout, timed_out; if ( opt_options.timeout ) { + /** + * Attempts to acquire a write lock on the specified name, executes the provided callback, + * and ensures the lock is released afterward. Supports options for timeout and handles + * multiple locks if the name parameter is an array. + * + * @param {string|string[]} name - The lock name(s) to acquire. + * @param {Object} [opt_options={}] - Optional configuration for the lock. + * @param {function} callback - The function to execute while holding the lock. + * @returns {Promise} The result of the callback execution. + * @throws {Error} If the lock acquisition times out. + */ timeout = setTimeout(() => { handle.unlock(); // TODO: verbose log option by service diff --git a/src/backend/src/services/MakeProdDebuggingLessAwfulService.js b/src/backend/src/services/MakeProdDebuggingLessAwfulService.js index 04295868f7..452ed5d7f5 100644 --- a/src/backend/src/services/MakeProdDebuggingLessAwfulService.js +++ b/src/backend/src/services/MakeProdDebuggingLessAwfulService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"claude"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -18,7 +19,6 @@ */ const { Context } = require("../util/context"); const BaseService = require("./BaseService"); -const { stringify_log_entry } = require("./runtime-analysis/LogService"); /** * This service registers a middleware that will apply the value of @@ -27,23 +27,61 @@ const { stringify_log_entry } = require("./runtime-analysis/LogService"); * Consequentially, the value of X-PUTER-DEBUG will included in all * log messages produced by the request. */ +/** +* @class MakeProdDebuggingLessAwfulService +* @extends BaseService +* @description Service that improves production debugging by capturing and processing debug headers. +* Extends the base service to provide middleware functionality that captures the X-PUTER-DEBUG header +* value and incorporates it into the request's Context object. This enables detailed logging and +* debugging capabilities in production environments while maintaining system security. The service +* also handles the creation and management of debug-specific log files for better traceability. +*/ class MakeProdDebuggingLessAwfulService extends BaseService { + static USE = { + logutil: 'core.util.logutil', + } static MODULES = { fs: require('fs'), } + /** + * Inner class that defines the modules required by the MakeProdDebuggingLessAwfulService. + * Currently includes the file system (fs) module for writing debug logs to files. + * @static + * @memberof MakeProdDebuggingLessAwfulService + */ static ProdDebuggingMiddleware = class ProdDebuggingMiddleware { + /** + * Middleware class that handles production debugging functionality + * by capturing and processing the X-PUTER-DEBUG header value. + * + * This middleware extracts the debug header value and makes it + * available through the Context for logging and debugging purposes. + */ constructor () { this.header_name_ = 'x-puter-debug'; } install (app) { app.use(this.run.bind(this)); } + /** + * Installs the middleware into the Express application + * + * @param {Express} app - The Express application instance + * @returns {void} + */ async run (req, res, next) { const x = Context.get(); x.set('prod-debug', req.headers[this.header_name_]); next(); } } + /** + * Handles the actual middleware execution for production debugging + * @param {Object} req - Express request object containing headers + * @param {Object} res - Express response object + * @param {Function} next - Express next middleware function + * @returns {Promise} + */ async _init () { // Initialize express middleware this.mw = new this.constructor.ProdDebuggingMiddleware(); @@ -66,7 +104,7 @@ class MakeProdDebuggingLessAwfulService extends BaseService { try { await this.modules.fs.promises.appendFile( outfile, - stringify_log_entry(log_details) + '\n', + this.logutil.stringify_log_entry(log_details) + '\n', ); } catch ( e ) { console.error(e); @@ -81,6 +119,13 @@ class MakeProdDebuggingLessAwfulService extends BaseService { }; }); } + /** + * Handles installation of the context-aware middleware for production debugging + * @param {*} _ Unused parameter + * @param {Object} options Installation options + * @param {Express} options.app Express application instance + * @returns {Promise} + */ async ['__on_install.middlewares.context-aware'] (_, { app }) { // Add express middleware this.mw.install(app); diff --git a/src/backend/src/services/NotificationService.js b/src/backend/src/services/NotificationService.js index 3be1d189a2..8101d8d95f 100644 --- a/src/backend/src/services/NotificationService.js +++ b/src/backend/src/services/NotificationService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -18,8 +19,9 @@ */ const APIError = require("../api/APIError"); const auth2 = require("../middleware/auth2"); +const { Context } = require("../util/context"); const { Endpoint } = require("../util/expressutil"); -const { TeePromise } = require("../util/promise"); +const { TeePromise } = require('@heyputer/putility').libs.promise; const BaseService = require("./BaseService"); const { DB_WRITE } = require("./database/consts"); @@ -33,16 +35,62 @@ const UserIDNotifSelector = user_id => async (self) => { return [user_id]; }; + +/** +* @class NotificationService +* @extends BaseService +* +* The NotificationService class is responsible for managing notifications within the application. +* It handles creating, storing, and sending notifications to users, as well as updating the status of notifications +* (e.g., marking them as read or acknowledged). +* +* @property {Object} MODULES - Static object containing modules used by the service, such as uuidv4 and express. +* @property {Object} merged_on_user_connected_ - Object to track connected users and manage delayed actions. +* @property {Object} notifs_pending_write - Object to track pending write operations for notifications. +* +* @method _construct - Initializes the service's internal state. +* @method _init - Initializes the service, setting up database connections and event listeners. +* @method __on_install.routes - Registers API routes for notification-related endpoints. +* @method on_user_connected - Handles actions when a user connects to the application. +* @method do_on_user_connected - Queries and updates unread notifications for a connected user. +* @method on_sent_to_user - Updates the status of a notification when it is sent to a user. +* @method notify - Sends a notification to a list of users and persists it in the database. +* +* @example +* const notificationService = new NotificationService(); +* notificationService.notify(UsernameNotifSelector('user123'), { +* source: 'notification-testing', +* icon_source: 'builtin', +* icon: 'logo.svg', +* title: 'Test Notification', +* text: 'This is a test notification.' +* }); +*/ class NotificationService extends BaseService { static MODULES = { uuidv4: require('uuid').v4, express: require('express'), } + + /** + * Initializes the NotificationService instance. + * This method sets up the initial state of the service, including any necessary + * data structures or configurations. + * + * @private + */ _construct () { this.merged_on_user_connected_ = {}; } + + /** + * Initializes the NotificationService by setting up necessary services, + * registering event listeners, and preparing the database connection. + * This method is called once during the service's lifecycle. + * @returns {Promise} A promise that resolves when initialization is complete. + */ async _init () { const svc_database = this.services.get('database'); this.db = svc_database.get(DB_WRITE, 'notification'); @@ -116,13 +164,44 @@ class NotificationService extends BaseService { }); } + + /** + * Handles the event when a user connects. + * + * This method checks if there is a timeout set for the user's connection event and clears it if it exists. + * If not, it sets a timeout to call `do_on_user_connected` after 2000 milliseconds. + * + * @param {object} params - The parameters object containing user data. + * @param {object} params.user - The user object with a `uuid` property. + * + * @returns {void} + */ async on_user_connected ({ user }) { if ( this.merged_on_user_connected_[user.uuid] ) { clearTimeout(this.merged_on_user_connected_[user.uuid]); } this.merged_on_user_connected_[user.uuid] = + /** + * Schedules the `do_on_user_connected` method to be called after a delay. + * + * This method sets a timer to call `do_on_user_connected` after 2000 milliseconds. + * If a timer already exists for the user, it clears the existing timer before setting a new one. + * + * @param {Object} params - The parameters object. + * @param {Object} params.user - The user object containing the user's UUID. + * @returns {Promise} A promise that resolves when the timer is set. + */ setTimeout(() => this.do_on_user_connected({ user }), 2000); } + /** + * Handles the event when a user connects. + * Sets a timeout to delay the execution of the `do_on_user_connected` method by 2 seconds. + * This helps in merging multiple events that occur in a short period. + * + * @param {Object} obj - The event object containing user information. + * @param {Object} obj.user - The user object with a `uuid` property. + * @async + */ async do_on_user_connected ({ user }) { // query the users unread notifications const notifications = await this.db.read( @@ -144,6 +223,14 @@ class NotificationService extends BaseService { for ( const n of notifications ) { n.value = this.db.case({ mysql: () => n.value, + /** + * Adjusts the value of a notification based on the database type. + * + * This method modifies the value of a notification to be JSON parsed + * if the database is not MySQL. + * + * @returns {Object} The adjusted notification value. + */ otherwise: () => JSON.parse(n.value ?? '{}'), })(); } @@ -166,6 +253,20 @@ class NotificationService extends BaseService { }); } + + /** + * Handles the action when a notification is sent to a user. + * + * This method is triggered when a notification is sent to a user, + * updating the notification's status to 'shown' in the database. + * It logs the user ID and response, updates the 'shown' timestamp, + * and ensures the notification is written to the database. + * + * @param {Object} params - The parameters containing the user ID and response. + * @param {number} params.user_id - The ID of the user receiving the notification. + * @param {Object} params.response - The response object containing the notification details. + * @param {string} params.response.uid - The unique identifier of the notification. + */ async on_sent_to_user ({ user_id, response }) { console.log('GOT IT AND IT WORKED!!!', user_id, response); const shown_ts = Math.floor(Date.now() / 1000); @@ -180,6 +281,17 @@ class NotificationService extends BaseService { ])); } + + /** + * Sends a notification to specified users. + * + * This method sends a notification to a list of users determined by the provided selector. + * It generates a unique identifier for the notification, emits an event to notify the GUI, + * and inserts the notification into the database. + * + * @param {Function} selector - A function that takes the service instance and returns a list of user IDs. + * @param {Object} notification - The notification details to be sent. + */ async notify (selector, notification) { const uid = this.modules.uuidv4(); const svc_event = this.services.get('event'); diff --git a/src/backend/src/services/OperationTraceService.js b/src/backend/src/services/OperationTraceService.js index 7ade53f4d8..1260ac664f 100644 --- a/src/backend/src/services/OperationTraceService.js +++ b/src/backend/src/services/OperationTraceService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -23,8 +24,15 @@ const { OtelFeature } = require("../traits/OtelFeature"); const APIError = require("../api/APIError"); const { AssignableMethodsFeature } = require("../traits/AssignableMethodsFeature"); +// CONTEXT_KEY is used to create a unique context key for operation tracing +// and is utilized throughout the OperationTraceService to manage frames. const CONTEXT_KEY = Context.make_context_key('operation-trace'); + +/** +* @class OperationFrame +* @description The `OperationFrame` class represents a frame within an operation trace. It is designed to manage the state, attributes, and hierarchy of frames within an operational context. This class provides methods to set status, calculate effective status, add tags, attributes, messages, errors, children, and describe the frame. It also includes methods to recursively search through frames to find attributes and handle frame completion. +*/ class OperationFrame { constructor ({ parent, label, x }) { this.parent = parent; @@ -68,6 +76,12 @@ class OperationFrame { this.parent._calc_effective_status(); } } + /** + * Sets the status of the frame and updates the effective status. + * This method logs the status change and updates the parent frame's effective status if necessary. + * + * @param {Object} status - The new status to set. + */ _calc_effective_status () { for ( const child of this.children ) { if ( child.status === OperationFrame.FRAME_STATUS_STUCK ) { @@ -97,6 +111,17 @@ class OperationFrame { } } + + /** + * Gets the effective status of the operation frame. + * + * This method returns the effective status of the current operation frame, + * considering the statuses of its children. The effective status is the + * aggregated status of the frame and its children, reflecting the current + * progress or state of the operation. + * + * @return {Object} The effective status of the operation frame. + */ get status () { return this.effective_status_; } @@ -132,6 +157,12 @@ class OperationFrame { return this; } + + /** + * Recursively traverses the frame hierarchy to find the root frame. + * + * @returns {OperationFrame} The root frame of the current frame hierarchy. + */ get_root_frame () { let frame = this; while ( frame.parent ) { @@ -140,6 +171,13 @@ class OperationFrame { return frame; } + + /** + * Marks the operation frame as done. + * This method sets the status of the operation frame to 'done' and updates + * the effective status accordingly. It triggers a recalculation of the + * effective status for parent frames if necessary. + */ done () { this.status = OperationFrame.FRAME_STATUS_DONE; } @@ -161,6 +199,14 @@ class OperationFrame { const prefix_deep = '│ '; const prefix_deep_end = ' '; + + /** + * Recursively builds a string representation of the frame and its children. + * + * @param {boolean} show_tree - If true, includes the tree structure of child frames. + * @param {OperationFrame} highlight_frame - The frame to highlight in the output. + * @returns {string} - A string representation of the frame and its children. + */ const recurse = (frame, prefix) => { const children = frame.children; for ( let i = 0; i < children.length; i++ ) { @@ -178,6 +224,13 @@ class OperationFrame { } } + +/** +* @class OperationTraceService +* @classdesc The OperationTraceService class manages operation frames and their statuses. +* It provides methods to add frames, track their progress, and handle their completion. +* This service is essential for monitoring and logging the lifecycle of operations within the system. +*/ class OperationTraceService { constructor ({ services }) { this.log = services.get('log-service').create('operation-trace'); @@ -186,6 +239,19 @@ class OperationTraceService { this.ongoing = {}; } + + /** + * Adds a new operation frame to the trace. + * + * This method creates a new frame with the given label and context, + * and adds it to the ongoing operations. If a context is provided, + * it logs the context description. The frame is then added to the + * parent frame if one exists, and the frame's description is logged. + * + * @param {string} label - The label for the new operation frame. + * @param {?Object} [x] - The context for the operation frame. + * @returns {OperationFrame} The new operation frame. + */ async add_frame (label) { return this.add_frame_sync(label); } @@ -219,6 +285,16 @@ class OperationTraceService { } } + +/** +* @class BaseOperation +* @extends AdvancedBase +* @description The BaseOperation class extends AdvancedBase and serves as the foundation for +* operations within the system. It integrates various features such as context awareness, +* observability through OpenTelemetry (OtelFeature), and assignable methods. This class is +* designed to be extended by specific operation classes to provide a common structure and +* functionality for running and tracing operations. +*/ class BaseOperation extends AdvancedBase { static FEATURES = [ new ContextAwareFeature(), @@ -226,6 +302,18 @@ class BaseOperation extends AdvancedBase { new AssignableMethodsFeature(), ] + + /** + * Executes the operation with the provided values. + * + * This method initiates an operation frame within the context, sets the operation status to working, + * executes the `_run` method, and handles post-run logic. It also manages the status of child frames + * and handles errors, updating the frame's attributes accordingly. + * + * @param {Object} values - The values to be used in the operation. + * @returns {Promise<*>} - The result of the operation. + * @throws {Error} - If the frame is missing or any other error occurs during the operation. + */ async run (values) { this.values = values; @@ -247,6 +335,17 @@ class BaseOperation extends AdvancedBase { // Run operation in new context try { + /** + * Runs an operation within a new context. + * + * This method sets up a new operation frame, updates the context, and runs the + * operation. It handles the operation's lifecycle, logging, and error handling. + * + * @async + * @function run + * @param {Object} values - The values to be passed to the operation. + * @returns {Promise<*>} The result of the operation. + */ return await x.arun(async () => { const x = Context.get(); const operationTraceSvc = x.get('services').get('operationTrace'); @@ -283,6 +382,13 @@ class BaseOperation extends AdvancedBase { this.frame.attributes[key] = value; } + + /** + * Updates the checkpoint for the current operation frame. + * + * @param {string} name - The name of the checkpoint to set. + * @returns {void} + */ _post_run () { let any_async = false; for ( const child of this.frame.children ) { diff --git a/src/backend/src/services/PermissionAPIService.js b/src/backend/src/services/PermissionAPIService.js index 5835531157..02715a45c8 100644 --- a/src/backend/src/services/PermissionAPIService.js +++ b/src/backend/src/services/PermissionAPIService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"claude"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -22,11 +23,28 @@ const { Endpoint } = require("../util/expressutil"); const { whatis } = require("../util/langutil"); const BaseService = require("./BaseService"); + +/** +* @class PermissionAPIService +* @extends BaseService +* @description Service class that handles API endpoints for permission management, including user-app permissions, +* user-user permissions, and group management. Provides functionality for creating groups, managing group memberships, +* granting/revoking various types of permissions, and checking access control lists (ACLs). Implements RESTful +* endpoints for group operations like creation, adding/removing users, and listing groups. +*/ class PermissionAPIService extends BaseService { static MODULES = { express: require('express'), }; + + /** + * Installs routes for authentication and permission management into the Express app + * @param {Object} _ Unused parameter + * @param {Object} options Installation options + * @param {Express} options.app Express application instance to install routes on + * @returns {Promise} + */ async ['__on_install.routes'] (_, { app }) { app.use(require('../routers/auth/get-user-app-token')) app.use(require('../routers/auth/grant-user-app')) @@ -44,6 +62,11 @@ class PermissionAPIService extends BaseService { }).attach(app); // track: scoping iife + /** + * Creates a scoped router for group-related endpoints using an IIFE pattern + * @private + * @returns {express.Router} Express router instance with isolated require scope + */ const r_group = (() => { const require = this.require; const express = require('express'); diff --git a/src/backend/src/services/ProtectedAppService.js b/src/backend/src/services/ProtectedAppService.js index 17adfe67a3..fea585d74f 100644 --- a/src/backend/src/services/ProtectedAppService.js +++ b/src/backend/src/services/ProtectedAppService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"mistral","model":"mistral-large-latest"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -21,7 +22,28 @@ const { UserActorType } = require("./auth/Actor"); const { PermissionImplicator, PermissionUtil, PermissionRewriter } = require("./auth/PermissionService"); const BaseService = require("./BaseService"); + +/** +* @class ProtectedAppService +* @extends BaseService +* @classdesc This class represents a service that handles protected applications. It extends the BaseService and includes +* methods for initializing permissions and registering rewriters and implicators for permission handling. The class +* ensures that the owner of a protected app has implicit permission to access it. +*/ class ProtectedAppService extends BaseService { + /** + * Class representing a service for protected applications. + * Extends the BaseService class to provide additional functionality specific to protected apps. + */ + + /** + * Initializes the ProtectedAppService. + * Registers a permission rewriter and implicator to handle application-specific permissions. + * @async + * @method _init + * @memberof ProtectedAppService + * @returns {Promise} A promise that resolves when the initialization is complete. + */ async _init () { const svc_permission = this.services.get('permission'); diff --git a/src/backend/src/services/PuterAPIService.js b/src/backend/src/services/PuterAPIService.js index d8a7f53bef..01cff4bb8a 100644 --- a/src/backend/src/services/PuterAPIService.js +++ b/src/backend/src/services/PuterAPIService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"openai-completion","model":"gpt-4o-mini"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -18,7 +19,25 @@ */ const BaseService = require("./BaseService"); + +/** +* @class PuterAPIService +* @extends BaseService +* +* The PuterAPIService class is responsible for integrating various routes +* into the web server for the Puter application. It acts as a middleware +* support layer, providing necessary API endpoints for handling various +* functionality such as authentication, user management, and application +* operations. This class is designed to extend the core functionalities +* of BaseService, ensuring that all routes are properly configured and +* available for use. +*/ class PuterAPIService extends BaseService { + /** + * Sets up the routes for the Puter API service. + * This method registers various API endpoints with the web server. + * It does not return a value as it configures the server directly. + */ async ['__on_install.routes'] () { const { app } = this.services.get('web-server'); diff --git a/src/backend/src/services/PuterHomepageService.js b/src/backend/src/services/PuterHomepageService.js index 754b4de715..a1a7b4b6e0 100644 --- a/src/backend/src/services/PuterHomepageService.js +++ b/src/backend/src/services/PuterHomepageService.js @@ -1,3 +1,4 @@ +// METADATA // {"ai-commented":{"service":"openai-completion","model":"gpt-4o"}} /* * Copyright (C) 2024 Puter Technologies Inc. * @@ -24,16 +25,35 @@ const {is_valid_url} = require('../helpers'); * PuterHomepageService serves the initial HTML page that loads the Puter GUI * and all of its assets. */ +/** +* This class serves the initial HTML page that loads the Puter GUI and all of its assets. +* It extends the BaseService class to provide common functionality. +*/ class PuterHomepageService extends BaseService { static MODULES = { fs: require('node:fs'), } + + /** + * This method sets a parameter for the GUI. + * It takes a key and value as arguments and adds them to the `gui_params` object. + * + * @param {string} key - The key for the parameter. + * @param {any} val - The value for the parameter. + */ _construct () { this.service_scripts = []; this.gui_params = {}; } + + /** + * @description This method initializes the PuterHomepageService by loading the manifest file. + * It reads the manifest file located at the specified path and parses its JSON content. + * The parsed data is then assigned to the `manifest` property of the instance. + * @returns {Promise} A promise that resolves with the initialized PuterHomepageService instance. + */ async _init () { // Load manifest const config = this.global_config; @@ -56,6 +76,12 @@ class PuterHomepageService extends BaseService { this.gui_params[key] = val; } + + /** + * @description This method sets a GUI parameter. It allows you to assign a value to a key within the `gui_params` object. + * @param {string} key - The key for the parameter. + * @param {any} val - The value for the parameter. + */ async send ({ req, res }, meta, launch_options) { const config = this.global_config; @@ -137,8 +163,6 @@ class PuterHomepageService extends BaseService { }) { const require = this.require; const {encode} = require('html-entities'); - const path_ = require('path'); - const fs_ = require('fs'); const e = encode; @@ -149,7 +173,6 @@ class PuterHomepageService extends BaseService { company, canonical_url, social_media_image, - icon, } = meta; gui_params = { @@ -164,7 +187,6 @@ class PuterHomepageService extends BaseService { const asset_dir = env === 'dev' ? '/src' : '/dist' ; - // const asset_dir = '/dist'; gui_params.asset_dir = asset_dir; @@ -183,11 +205,17 @@ class PuterHomepageService extends BaseService { // set social media image to default if it is not valid const social_media_image_url = social_media_image || `${asset_dir}/images/screenshot.png`; - const writeScriptTag = path => - `\n` - ; + // Custom script tags to be added to the homepage by extensions + // an event is emitted to allow extensions to add their own script tags + // the event is emitted with an object containing a custom_script_tags array + // which extensions can push their script tags to + let custom_script_tags = []; + let custom_script_tags_str = ''; + process.emit('add_script_tags_to_homepage_html', { custom_script_tags }); + + for (const tag of custom_script_tags) { + custom_script_tags_str += tag; + } return ` @@ -238,6 +266,20 @@ class PuterHomepageService extends BaseService { + ${ + custom_script_tags_str + } ${ use_bundled_gui ? `` @@ -279,6 +324,20 @@ class PuterHomepageService extends BaseService { + + @@ -163,7 +169,7 @@

Payout Method

-
+
diff --git a/src/dev-center/js/dev-center.js b/src/dev-center/js/dev-center.js index 8aa2f44540..b015601270 100644 --- a/src/dev-center/js/dev-center.js +++ b/src/dev-center/js/dev-center.js @@ -54,6 +54,7 @@ const loading_spinner = `HTML, JS, CSS, ...

`; const index_missing_error = `Please upload an 'index.html' file or if you're uploading a directory, make sure it contains an 'index.html' file at its root.`; const lock_svg = ''; +const copy_svg = ` `; // authUsername (async () => { @@ -518,9 +519,11 @@ function generate_edit_app_section(app) {
-
+
-
App has been successfully updated.×
+
App has been successfully updated.× +

Give it a try!

+

Basic

@@ -534,7 +537,9 @@ function generate_edit_app_section(app) { - +
+ ${copy_svg} +
@@ -555,8 +560,8 @@ function generate_edit_app_section(app) { -

A comma-separated list of file type specifiers. For example if you include .txt, your apps could be opened when a user clicks on a TXT file.

- +

A list of file type specifiers. For example if you include .txt your apps could be opened when a user clicks on a TXT file.

+

Window

@@ -612,9 +617,10 @@ function generate_edit_app_section(app) {

credentialless attribute for the iframe tag.

-
- - +
+ + +
` @@ -762,6 +768,63 @@ async function edit_app_section(cur_app_name) { toggleResetButton(); // Ensure Reset button is initially disabled $('#edit-app').show(); + const filetype_association_input = document.querySelector('textarea[id=edit-app-filetype-associations]'); + let tagify = new Tagify(filetype_association_input, { + pattern: /\.(?:[a-z0-9]+)|(?:[a-z]+\/(?:[a-z0-9.-]+|\*))/, + delimiters: ", ", + enforceWhitelist: false, + dropdown : { + // show the dropdown immediately on focus (0 character typed) + enabled: 0, + }, + whitelist: [ + // MIME type patterns + "text/*", "image/*", "audio/*", "video/*", "application/*", + + // Documents + ".doc", ".docx", ".pdf", ".txt", ".odt", ".rtf", ".tex", ".md", ".pages", ".epub", ".mobi", ".azw", ".azw3", ".djvu", ".xps", ".oxps", ".fb2", ".textile", ".markdown", ".asciidoc", ".rst", ".wpd", ".wps", ".abw", ".zabw", + + // Spreadsheets + ".xls", ".xlsx", ".csv", ".ods", ".numbers", ".tsv", ".gnumeric", ".xlt", ".xltx", ".xlsm", ".xltm", ".xlam", ".xlsb", + + // Presentations + ".ppt", ".pptx", ".key", ".odp", ".pps", ".ppsx", ".pptm", ".potx", ".potm", ".ppam", + + // Images + ".jpg", ".jpeg", ".png", ".gif", ".bmp", ".tiff", ".tif", ".svg", ".webp", ".ico", ".psd", ".ai", ".eps", ".raw", ".cr2", ".nef", ".orf", ".sr2", ".heic", ".heif", ".avif", ".jxr", ".hdp", ".wdp", ".jng", ".xcf", ".pgm", ".pbm", ".ppm", ".pnm", + + // Video + ".mp4", ".avi", ".mov", ".wmv", ".mkv", ".flv", ".webm", ".m4v", ".mpeg", ".mpg", ".3gp", ".3g2", ".ogv", ".vob", ".drc", ".gifv", ".mng", ".qt", ".yuv", ".rm", ".rmvb", ".asf", ".amv", ".m2v", ".svi", + + // Audio + ".mp3", ".wav", ".aac", ".flac", ".ogg", ".m4a", ".wma", ".aiff", ".alac", ".ape", ".au", ".mid", ".midi", ".mka", ".pcm", ".ra", ".ram", ".snd", ".wv", ".opus", + + // Code/Development + ".js", ".ts", ".html", ".css", ".json", ".xml", ".php", ".py", ".java", ".cpp", ".c", ".cs", ".h", ".hpp", ".hxx", ".rs", ".go", ".rb", ".pl", ".swift", ".kt", ".kts", ".scala", ".coffee", ".sass", ".scss", ".less", ".jsx", ".tsx", ".vue", ".sh", ".bash", ".zsh", ".fish", ".ps1", ".bat", ".cmd", ".sql", ".r", ".dart", ".f", ".f90", ".for", ".lua", ".m", ".mm", ".clj", ".erl", ".ex", ".exs", ".elm", ".hs", ".lhs", ".lisp", ".ml", ".mli", ".nim", ".pl", ".rkt", ".v", ".vhd", + + // Archives + ".zip", ".rar", ".7z", ".tar", ".gz", ".bz2", ".xz", ".z", ".lz", ".lzma", ".tlz", ".txz", ".tgz", ".tbz2", ".bz", ".br", ".lzo", ".ar", ".cpio", ".shar", ".lrz", ".lz4", ".lz2", ".rz", ".sfark", ".sz", ".zoo", + + // Database + ".db", ".sql", ".sqlite", ".sqlite3", ".dbf", ".mdb", ".accdb", ".db3", ".s3db", ".dbx", + + // Fonts + ".ttf", ".otf", ".woff", ".woff2", ".eot", ".pfa", ".pfb", ".sfd", + + // CAD and 3D + ".dwg", ".dxf", ".stl", ".obj", ".fbx", ".dae", ".3ds", ".blend", ".max", ".ma", ".mb", ".c4d", ".skp", ".usd", ".usda", ".usdc", ".abc", + + // Scientific/Technical + ".mat", ".fig", ".nb", ".cdf", ".fits", ".fts", ".fit", ".gmsh", ".msh", ".fem", ".neu", ".hdf", ".h5", ".nx", ".unv", + + // System + ".exe", ".dll", ".so", ".dylib", ".app", ".dmg", ".iso", ".img", ".bin", ".msi", ".apk", ".ipa", ".deb", ".rpm", + + // Directory + ".directory" + ], + }) + // -------------------------------------------------------- // Dragster // -------------------------------------------------------- @@ -1102,6 +1165,31 @@ $(document).on('click', '.edit-app-save-btn', async function (e) { } } + // parse filetype_associations + if(filetype_associations !== ''){ + filetype_associations = JSON.parse(filetype_associations); + filetype_associations = filetype_associations.map((type) => { + const fileType = type.value; + if ( + !fileType || + fileType === "." || + fileType === "/" + ) { + error = `File Association Type must be valid.`; + return null; // Return null for invalid cases + } + const lower = fileType.toLocaleLowerCase(); + + if (fileType.includes("/")) { + return lower; + } else if (fileType.includes(".")) { + return "." + lower.split(".")[1]; + } else { + return "." + lower; + } + }).filter(Boolean); + } + // error? if (error) { $('#edit-app-error').show(); @@ -1113,8 +1201,8 @@ $(document).on('click', '.edit-app-save-btn', async function (e) { // show working spinner puter.ui.showSpinner(); - // parse filetype_associations - filetype_associations = filetype_associations.split(',').map(element => element.trim()); + + // disable submit button $('.edit-app-save-btn').prop('disabled', true); @@ -1638,7 +1726,123 @@ function sort_apps() { } } +/** + * Checks if the items being deployed contain a .git directory + * @param {Array|string} items - Items to check (can be path string or array of items) + * @returns {Promise} - True if .git directory is found + */ +async function hasGitDirectory(items) { + // Case 1: Single Puter path + if (typeof items === 'string' && (items.startsWith('/') || items.startsWith('~'))) { + const stat = await puter.fs.stat(items); + if (stat.is_dir) { + const files = await puter.fs.readdir(items); + return files.some(file => file.name === '.git' && file.is_dir); + } + return false; + } + + // Case 2: Array of Puter items + if (Array.isArray(items) && items[0]?.uid) { + return items.some(item => item.name === '.git' && item.is_dir); + } + + // Case 3: Local items (DataTransferItems) + if (Array.isArray(items)) { + for (let item of items) { + if (item.fullPath?.includes('/.git/') || + item.path?.includes('/.git/') || + item.filepath?.includes('/.git/')) { + return true; + } + } + } + + return false; +} + +/** + * Shows a warning dialog about .git directory deployment + * @returns {Promise} - True if the user wants to proceed with deployment + */ +async function showGitWarningDialog() { + try { + // Check if the user has chosen to skip the warning + const skipWarning = await puter.kv.get('skip-git-warning'); + + // Log retrieved value for debugging + console.log('Retrieved skip-git-warning:', skipWarning); + + // If the user opted to skip the warning, proceed without showing it + if (skipWarning === true) { + return true; + } + } catch (error) { + console.error('Error accessing KV store:', error); + // If KV store access fails, fall back to showing the dialog + } + + // Create the modal dialog + const modal = document.createElement('div'); + modal.innerHTML = ` +
+

Warning: Git Repository Detected

+

A .git directory was found in your deployment files. Deploying .git directories may:

+
    +
  • Expose sensitive information like commit history and configuration
  • +
  • Significantly increase deployment size
  • +
+
+ + +
+
+ + +
+
+
+ `; + document.body.appendChild(modal); + + return new Promise((resolve) => { + // Handle "Continue Deployment" + document.getElementById('continue-deployment').addEventListener('click', async () => { + try { + const skipChecked = document.getElementById('skip-git-warning')?.checked; + if (skipChecked) { + console.log("Saving 'skip-git-warning' preference as true"); + await puter.kv.set('skip-git-warning', true); + } + } catch (error) { + console.error('Error saving user preference to KV store:', error); + } finally { + document.body.removeChild(modal); + resolve(true); // Continue deployment + } + }); + + // Handle "Cancel Deployment" + document.getElementById('cancel-deployment').addEventListener('click', () => { + document.body.removeChild(modal); + resolve(false); // Cancel deployment + }); + }); +} + window.deploy = async function (app, items) { + // Check for .git directory before proceeding + try { + if (await hasGitDirectory(items)) { + const shouldProceed = await showGitWarningDialog(); + if (!shouldProceed) { + reset_drop_area(); + return; + } + } + } catch (err) { + console.error('Error checking for .git directory:', err); + } let appdata_dir, current_app_dir; // disable deploy button @@ -2564,3 +2768,13 @@ async function handleSocialImageUpload(app_name, socialImageData) { throw err; } } + +$(document).on('click', '.copy-app-uid', function(e) { + const appUID = $('#edit-app-uid').val(); + navigator.clipboard.writeText(appUID); + // change to 'copied' + $(this).html('Copied'); + setTimeout(() => { + $(this).html(copy_svg); + }, 2000); +}); \ No newline at end of file diff --git a/src/gui/src/UI/UIDesktop.js b/src/gui/src/UI/UIDesktop.js index 20846955b5..6b9d619fb6 100644 --- a/src/gui/src/UI/UIDesktop.js +++ b/src/gui/src/UI/UIDesktop.js @@ -163,6 +163,10 @@ async function UIDesktop(options){ $(`.window[data-path="${html_encode(window.trash_path)}" i]`).find('.item-container').empty(); }) + /** + * This event is triggered if a user receives a notification during + * an active session. + */ window.socket.on('notif.message', async ({ uid, notification }) => { let icon = window.icons[notification.icon]; let round_icon = false; @@ -207,6 +211,14 @@ async function UIDesktop(options){ }); }); + /** + * This event is triggered at the beginning of the session, after a websocket + * connection is established, because the backend informs the frontend of all + * unread notifications. + * + * It is not necessary to query unreads separately. If this stops working, + * then this event should be fixed rather than querying unreads separately. + */ window.__already_got_unreads = false; window.socket.on('notif.unreads', async ({ unreads }) => { if ( window.__already_got_unreads ) return; @@ -643,6 +655,8 @@ async function UIDesktop(options){ >`; h += `
`; + h += ``; + // Get window sidebar width puter.kv.get('window_sidebar_width').then(async (val) => { let value = parseInt(val); @@ -1089,7 +1103,7 @@ async function UIDesktop(options){ // User options // ---------------------------------------------------- let ht = ''; - ht += `
`; + ht += `
`; // logo ht += ``; @@ -1098,10 +1112,8 @@ async function UIDesktop(options){ ht += ``; ht += `
`; - // 'show desktop' - if(window.is_fullpage_mode){ - ht += `Show Desktop `; - } + // 'Show Desktop' + ht += ``; // refer if(window.user.referral_code){ @@ -1155,7 +1167,12 @@ async function UIDesktop(options){ // get app metadata try{ window.app_launched_from_url = await puter.apps.get(window.url_paths[1]) - window.is_fullpage_mode = window.app_launched_from_url.metadata?.fullpage_on_landing ?? false; + window.is_fullpage_mode = window.app_launched_from_url.metadata?.fullpage_on_landing ?? window.is_fullpage_mode ?? false; + + // show 'Show Desktop' button + if(window.is_fullpage_mode){ + $('.show-desktop-btn').removeClass('hidden'); + } }catch(e){ console.error(e); } @@ -1288,55 +1305,6 @@ async function UIDesktop(options){ }) } - // fetch notifications - fetch(puter.APIOrigin + "/drivers/call", { - "headers": { - "Content-Type": "application/json", - "Authorization": `Bearer ${puter.authToken}`, - }, - "body": JSON.stringify({ - interface: 'puter-notifications', - method: 'select', - args: {} - }), - "method": "POST", - }) - .then(response => response.json()) - .then(data => { - if(data && data.result && data.result.length > 0){ - data.data?.forEach(async notification => { - let icon = window.icons['puter-logo.svg']; - let round_icon = false; - - if(notification.template === "file-shared-with-you" && notification.fields?.username){ - let profile_pic = await get_profile_picture(notification.fields?.username); - if(profile_pic){ - icon = profile_pic; - round_icon = true; - notification.round_icon = round_icon; - } - } - notification.icon = icon; - - notification.click = async (notif) => { - if(notification.template === "file-shared-with-you"){ - let item_path = '/' + notification.fields?.username; - UIWindow({ - path: '/' + notification.fields?.username, - title: path.basename(item_path), - icon: await item_icon({is_dir: true, path: item_path}), - is_dir: true, - app: 'explorer', - }); - } - } - - UINotification(notification); - }) - } - }) - - //-------------------------------------------------------------------------------------- // Trying to view a user's public folder? // i.e. https://puter.com/@ @@ -1809,8 +1777,6 @@ window.remove_taskbar_item = function(item){ $(item).animate({width: 0}, 200, function(){ $(item).remove(); }) - - window.recalibrate_taskbar_item_positions(); } window.enter_fullpage_mode = (el_window)=>{ diff --git a/src/gui/src/UI/UIPopover.js b/src/gui/src/UI/UIPopover.js index ed5a3cf9da..389c39a3a7 100644 --- a/src/gui/src/UI/UIPopover.js +++ b/src/gui/src/UI/UIPopover.js @@ -30,7 +30,7 @@ function UIPopover(options){ options.content = options.content ?? ''; let h = ''; - h += `
`; + h += `
`; h += options.content; h += `
`; diff --git a/src/gui/src/UI/UIPrompt.js b/src/gui/src/UI/UIPrompt.js index 56b8f4ea4c..2dadf6d474 100644 --- a/src/gui/src/UI/UIPrompt.js +++ b/src/gui/src/UI/UIPrompt.js @@ -81,6 +81,15 @@ function UIPrompt(options){ setTimeout(function(){ $(this_window).find('.prompt-input').get(0).focus({preventScroll:true}); }, 30); + + // Add event listener for Escape key + $(document).on('keyup.uiprompt', function(e) { + if (e.key === 'Escape') { + resolve(false); + $(el_window).close(); + $(document).off('keyup.uiprompt'); // Remove event listener + } + }); }, ...options.window_options, window_css:{ @@ -108,6 +117,7 @@ function UIPrompt(options){ resolve(false); } $(el_window).close(); + $(document).off('keyup.uiprompt'); // Remove event listener return false; }) diff --git a/src/gui/src/UI/UITaskbar.js b/src/gui/src/UI/UITaskbar.js index d0818e0e26..172243409e 100644 --- a/src/gui/src/UI/UITaskbar.js +++ b/src/gui/src/UI/UITaskbar.js @@ -42,7 +42,7 @@ async function UITaskbar(options){ }); let h = ''; - h += `
`; + h += `
`; $('.desktop').append(h); @@ -65,11 +65,12 @@ async function UITaskbar(options){ // show popover let popover = UIPopover({ - content: `
`, + content: `
`, snapToElement: item, parent_element: item, width: 500, height: 500, + class: 'popover-launcher', center_horizontally: true, }); @@ -238,7 +239,6 @@ async function UITaskbar(options){ } }) - window.recalibrate_taskbar_item_positions(); window.make_taskbar_sortable(); } @@ -305,28 +305,4 @@ window.make_taskbar_sortable = function(){ }); } -window.recalibrate_taskbar_item_positions = function(){ - // if this is mobile rearrange taskbar item positions based on absolute position - // taskbar items must be centered unless there is overflow. If there is overflow, the taskbar items must be left aligned - if(isMobile.phone || window.desktop_width < 800){ - let taskbar_items = $('.taskbar-item'); - let taskbar_width = taskbar_items.length * 60; - - if(taskbar_width > window.desktop_width){ - // set taskbar items to absolute position - $('.taskbar-item').css('position', 'absolute'); - $('.taskbar').css('display', 'block'); - // set left position for each taskbar item - let left = 0; - for (let index = 0; index < taskbar_items.length; index++) { - const taskbar_item = taskbar_items[index]; - $(taskbar_item).css('left', left); - left += 60; - } - }else{ - $('.taskbar-item').css('position', 'unset'); - $('.taskbar').css('justify-content', 'center'); - } - } -} export default UITaskbar; \ No newline at end of file diff --git a/src/gui/src/UI/UITaskbarItem.js b/src/gui/src/UI/UITaskbarItem.js index 31007ee9c5..93221d1a90 100644 --- a/src/gui/src/UI/UITaskbarItem.js +++ b/src/gui/src/UI/UITaskbarItem.js @@ -30,6 +30,8 @@ function UITaskbarItem(options){ options.open_windows_count = options.open_windows_count ?? 0; options.lock_keep_in_taskbar = options.lock_keep_in_taskbar ?? false; options.append_to_taskbar = options.append_to_taskbar ?? true; + options.before_trash = options.before_trash ?? false; + const element_id = window.global_element_id++; h += `
`; h += `
`; - if(options.append_to_taskbar) - $('.taskbar').append(h); - else + if(options.append_to_taskbar) { + if (options.before_trash){ + $('.taskbar-item[data-app="trash"]').before(h); + }else{ + $('.taskbar').append(h); + } + }else{ $('body').prepend(h); + } const el_taskbar_item = document.querySelector(`#taskbar-item-${tray_item_id}`); // fade in the taskbar item $(el_taskbar_item).show(50); - $(el_taskbar_item).on("click", function(){ + $(el_taskbar_item).on("click", function(e){ + e.preventDefault(); + e.stopPropagation(); + + // if this is for the launcher popover, and it's mobile, and has-open-popover, close the popover + if( $(el_taskbar_item).attr('data-name') === 'Start' + && (isMobile.phone || isMobile.tablet) && $(el_taskbar_item).hasClass('has-open-popover')){ + $('.popover').remove(); + return; + } + // If this item has an open context menu, don't do anything if($(el_taskbar_item).hasClass('has-open-contextmenu')) return; + el_taskbar_item.querySelector("img").animate( + [ + { transform: 'translateY(0) scale(1)' }, + { transform: 'translateY(-5px) scale(1.2)' }, + { transform: 'translateY(0) scale(1)' } + ], + { + duration: 300, + easing: 'ease-out', + } + ); + if(options.onClick === undefined || options.onClick(el_taskbar_item) === false){ // re-show each window in this app group $(`.window[data-app="${options.app}"]`).showWindow(); @@ -205,8 +234,7 @@ function UITaskbarItem(options){ menu_items.push({ html: i18n('show_all_windows'), onClick: function(){ - if(open_windows > 0) - $(el_taskbar_item).trigger('click'); + $(`.window[data-app="${options.app}"]`).showWindow(); } }) // ------------------------------------------- @@ -376,10 +404,6 @@ function UITaskbarItem(options){ } }); - if(options.append_to_taskbar){ - window.recalibrate_taskbar_item_positions(); - } - return el_taskbar_item; } diff --git a/src/gui/src/UI/UIWindow.js b/src/gui/src/UI/UIWindow.js index 0873a394fc..99bce7db0c 100644 --- a/src/gui/src/UI/UIWindow.js +++ b/src/gui/src/UI/UIWindow.js @@ -86,11 +86,14 @@ async function UIWindow(options) { options.is_maximized = options.is_maximized ?? false; options.is_openFileDialog = options.is_openFileDialog ?? false; options.is_resizable = options.is_resizable ?? true; + // if this is a fullpage window, it won't be resizable - if(options.is_fullpage) + if(options.is_fullpage){ + options.is_maximized = false; options.is_resizable = false; + } - // in the embedded/fullpage mode every window is on top since there is no taskbar to switch between windows + // In the embedded/fullpage mode every window is on top since there is no taskbar to switch between windows // if user has specifically asked for this window to NOT stay on top, honor it. if((window.is_embedded || window.is_fullpage_mode) && !options.parent_uuid && options.stay_on_top !== false) options.stay_on_top = true; @@ -151,7 +154,7 @@ async function UIWindow(options) { else if(isMobile.phone) options.top = 100; - if(isMobile.phone){ + if(isMobile.phone && !options.center && !options.dominant){ options.left = 0; options.top = window.toolbar_height + 'px'; options.width = '100%'; @@ -186,6 +189,7 @@ async function UIWindow(options) { user_set_url_params = '?'+ user_set_url_params.join('&'); } h += `
`; // iframe, for apps if(options.iframe_url || options.iframe_srcdoc){ - let allow_str = `camera; encrypted-media; gamepad; display-capture; geolocation; gyroscope; microphone; midi; clipboard-read; clipboard-write; fullscreen;`; + let allow_str = `camera; encrypted-media; gamepad; display-capture; geolocation; gyroscope; microphone; midi; clipboard-read; clipboard-write; fullscreen; web-share;`; if(window.co_isolation_enabled) allow_str += ' cross-origin-isolated;'; //