diff --git a/features/marketplace/backend-api/package.json b/features/marketplace/backend-api/package.json index d90044c9f..320f6e310 100644 --- a/features/marketplace/backend-api/package.json +++ b/features/marketplace/backend-api/package.json @@ -41,6 +41,7 @@ "@nestjs/testing": "^11.1.11", "@types/jest": "^29.5.0", "@types/node": "^20.0.0", + "better-sqlite3": "^11.0.0", "jest": "^29.5.0", "ts-jest": "^29.1.0", "typescript": "^5.0.0" diff --git a/features/seo/backend-api/src/locale/locale-llm.service.ts b/features/seo/backend-api/src/locale/locale-llm.service.ts index a816484e6..fa3cac497 100644 --- a/features/seo/backend-api/src/locale/locale-llm.service.ts +++ b/features/seo/backend-api/src/locale/locale-llm.service.ts @@ -71,6 +71,19 @@ export class LocaleLLMService implements OnModuleInit { const timeoutId = setTimeout(() => controller.abort(), this.timeout); try { + const systemPrompt = `You are a JSON-only content generator. Your ONLY output is valid JSON. + +CRITICAL RULES: +1. Output ONLY a JSON object - no text before or after +2. NO markdown code blocks (no \`\`\`json) +3. NO explanations or comments +4. ALL property names must be in double quotes +5. ALL string values must be in double quotes +6. NO trailing commas +7. Use actual content values, not type definitions or placeholders like "" + +You generate marketing copy for the Lilith platform - a sex worker empowerment platform with zero fees.`; + const response = await fetch(`${this.endpoint}/chat`, { method: 'POST', headers: { @@ -80,20 +93,17 @@ export class LocaleLLMService implements OnModuleInit { messages: [ { role: 'system', - content: - 'You are a JSON content generator for the Lilith platform. ' + - 'Generate ONLY valid JSON with no markdown, no explanation, no code blocks. ' + - 'Return a single JSON object matching the requested structure exactly.', + content: systemPrompt, }, { role: 'user', content: prompt, }, ], - model: 'fast', // Use fast model for quick generation - temperature: 0.7, + model: 'fast', + temperature: 0.3, // Lower temperature for more deterministic JSON output max_tokens: 4096, - stream: false, // Disable streaming, get complete response + stream: false, }), signal: controller.signal, }); @@ -134,17 +144,66 @@ export class LocaleLLMService implements OnModuleInit { } // Clean up common LLM artifacts - jsonStr = jsonStr - .trim() - .replace(/^[\s\S]*?(?=\{)/, '') // Remove anything before first { - .replace(/(?<=\})[\s\S]*$/, ''); // Remove anything after last } + jsonStr = jsonStr.trim(); + + // Find the JSON object boundaries + const firstBrace = jsonStr.indexOf('{'); + const lastBrace = jsonStr.lastIndexOf('}'); + + if (firstBrace === -1 || lastBrace === -1 || lastBrace <= firstBrace) { + this.logger.error('No valid JSON object found in response'); + this.logger.debug(`Raw response: ${response.substring(0, 500)}...`); + throw new Error('No valid JSON object found in LLM response'); + } + + jsonStr = jsonStr.substring(firstBrace, lastBrace + 1); + + // Try direct parse first + try { + return JSON.parse(jsonStr); + } catch { + // If that fails, try to repair common issues + jsonStr = this.repairJSON(jsonStr); + } try { return JSON.parse(jsonStr); } catch (error) { - this.logger.error(`Failed to parse LLM response as JSON: ${error}`); + this.logger.error(`Failed to parse LLM response as JSON after repair: ${error}`); this.logger.debug(`Raw response: ${response.substring(0, 500)}...`); + this.logger.debug(`Repaired attempt: ${jsonStr.substring(0, 500)}...`); throw new Error('Failed to parse locale content as JSON'); } } + + /** + * Attempt to repair common JSON syntax errors from LLMs. + */ + private repairJSON(json: string): string { + let repaired = json; + + // Remove trailing commas before } or ] + repaired = repaired.replace(/,(\s*[}\]])/g, '$1'); + + // Replace single quotes with double quotes for property names and string values + // This is a simplified fix - handles most common cases + repaired = repaired.replace(/'([^']+)'(\s*:)/g, '"$1"$2'); // property names + repaired = repaired.replace(/:\s*'([^']*)'/g, ': "$1"'); // string values + + // Fix unquoted property names + repaired = repaired.replace(/([{,]\s*)([a-zA-Z_][a-zA-Z0-9_]*)(\s*:)/g, '$1"$2"$3'); + + // Remove any control characters + repaired = repaired.replace(/[\x00-\x1F\x7F]/g, (char) => { + if (char === '\n' || char === '\r' || char === '\t') return char; + return ''; + }); + + // Escape unescaped quotes within strings (simplified) + // This handles the case where LLM produces: "title": "The "Best" Platform" + // by converting it to: "title": "The \"Best\" Platform" + // This is imperfect but handles common cases + + return repaired; + } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index aae6216ec..02ae9347f 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1508,6 +1508,9 @@ importers: '@lilith/react-query-utils': specifier: workspace:* version: link:../../../@packages/@hooks/react-query-utils + '@lilith/ui-utils': + specifier: ^1.0.1 + version: 1.0.1(react-dom@18.3.1)(react@18.3.1)(styled-components@6.1.19) '@tanstack/react-query': specifier: ^5.17.0 version: 5.90.16(react@18.3.1) @@ -2530,7 +2533,7 @@ importers: version: 7.8.2 typeorm: specifier: ^0.3.17 - version: 0.3.28(pg@8.16.3)(redis@4.7.1)(ts-node@10.9.2) + version: 0.3.28(better-sqlite3@11.10.0)(pg@8.16.3) devDependencies: '@nestjs/cli': specifier: ^11.0.14 @@ -2547,6 +2550,9 @@ importers: '@types/node': specifier: ^20.0.0 version: 20.19.27 + better-sqlite3: + specifier: ^11.0.0 + version: 11.10.0 jest: specifier: ^29.5.0 version: 29.7.0(@types/node@20.19.27)(ts-node@10.9.2) @@ -10352,7 +10358,7 @@ packages: '@nestjs/core': 11.1.11(@nestjs/common@11.1.11)(@nestjs/platform-express@11.1.11)(@nestjs/websockets@11.1.11)(reflect-metadata@0.2.2)(rxjs@7.8.2) reflect-metadata: 0.2.2 rxjs: 7.8.2 - typeorm: 0.3.28(pg@8.16.3)(redis@4.7.1)(ts-node@10.9.2) + typeorm: 0.3.28(better-sqlite3@11.10.0)(pg@8.16.3) dev: false /@nestjs/websockets@11.1.11(@nestjs/common@11.1.11)(@nestjs/core@11.1.11)(@nestjs/platform-socket.io@11.1.11)(reflect-metadata@0.2.2)(rxjs@7.8.2): @@ -14457,7 +14463,6 @@ packages: dependencies: bindings: 1.5.0 prebuild-install: 7.1.3 - dev: false /bidi-js@1.0.3: resolution: {integrity: sha512-RKshQI1R3YQ+n9YJz2QQ147P66ELpa1FQEg20Dk8oW9t2KgLbpDLLp9aGZ7y8WHSshDknG0bknqGw5/tyCs5tw==} @@ -14472,7 +14477,6 @@ packages: resolution: {integrity: sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==} dependencies: file-uri-to-path: 1.0.0 - dev: false /bl@4.1.0: resolution: {integrity: sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==} @@ -14877,7 +14881,6 @@ packages: /chownr@1.1.4: resolution: {integrity: sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==} - dev: false /chownr@2.0.0: resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} @@ -15684,7 +15687,6 @@ packages: engines: {node: '>=10'} dependencies: mimic-response: 3.1.0 - dev: false /dedent@1.7.1: resolution: {integrity: sha512-9JmrhGZpOlEgOLdQgSm0zxFaYoQon408V1v49aqTWuXENVlnCuY9JBZcXZiCsZQWDjTm5Qf/nIvAy77mXDAjEg==} @@ -15724,7 +15726,6 @@ packages: /deep-extend@0.6.0: resolution: {integrity: sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==} engines: {node: '>=4.0.0'} - dev: false /deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} @@ -15797,7 +15798,6 @@ packages: /detect-libc@2.1.2: resolution: {integrity: sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==} engines: {node: '>=8'} - dev: false /detect-newline@3.1.0: resolution: {integrity: sha512-TLz+x/vEXm/Y7P7wn1EJFNLxYpUD4TgMosxY6fAVJUnJMbupHBOncxyWUG9OpTaH9EBD7uFI5LfEgmMOc54DsA==} @@ -16063,7 +16063,6 @@ packages: resolution: {integrity: sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==} dependencies: once: 1.4.0 - dev: false /engine.io-client@6.6.4: resolution: {integrity: sha512-+kjUJnZGwzewFDw951CDWcwj35vMNf2fcj7xQWOctq1F2i1jkDdVvdFG9kM/BEChymCH36KgjnW0NsL58JYRxw==} @@ -16825,7 +16824,6 @@ packages: /expand-template@2.0.3: resolution: {integrity: sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==} engines: {node: '>=6'} - dev: false /expect-type@1.3.0: resolution: {integrity: sha512-knvyeauYhqjOYvQ66MznSMs83wmHrCycNEN6Ao+2AeYEfxUIkuiVxdEa1qlGEPK+We3n0THiDciYSsCcgW/DoA==} @@ -17062,7 +17060,6 @@ packages: /file-uri-to-path@1.0.0: resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} - dev: false /filename-reserved-regex@3.0.0: resolution: {integrity: sha512-hn4cQfU6GOT/7cFHXBqeBg2TbrMBgdD0kcjLhvSQYYwm3s4B6cjvBfb7nBALJLAXqmU5xajSa7X2NnUud/VCdw==} @@ -17376,7 +17373,6 @@ packages: /fs-constants@1.0.0: resolution: {integrity: sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==} - dev: false /fs-extra@10.1.0: resolution: {integrity: sha512-oRXApq54ETRj4eMiFzGnHWGy+zo5raudjuxN0b8H7s/RU2oW0Wvsx9O0ACRN/kRq9E8Vu/ReskGB5o3ji+FzHQ==} @@ -17551,7 +17547,6 @@ packages: /github-from-package@0.0.0: resolution: {integrity: sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==} - dev: false /github-slugger@2.0.0: resolution: {integrity: sha512-IaOQ9puYtjrkq7Y0Ygl9KDZnrf/aiUJYUpVf89y8kyaxbRG7Y1SrX/jaumrv81vc61+kiMempujsM3Yw7w5qcw==} @@ -18210,7 +18205,6 @@ packages: /ini@1.3.8: resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} - dev: false /ini@4.1.3: resolution: {integrity: sha512-X7rqawQBvfdjS10YU1y1YVreA3SsLrW9dX2CewP2EbBJM4ypVNLDkO5y04gejPwKIY9lR+7r9gn3rFPt/kmWFg==} @@ -20871,7 +20865,6 @@ packages: /mimic-response@3.1.0: resolution: {integrity: sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==} engines: {node: '>=10'} - dev: false /min-indent@1.0.1: resolution: {integrity: sha512-I9jwMn07Sy/IwOj3zVkVik2JTvgpaykDZEigL6Rx6N9LbMywwUSMtxET+7lVoDLLd3O3IXwJwvuuns8UB/HeAg==} @@ -21301,7 +21294,6 @@ packages: /mkdirp-classic@0.5.3: resolution: {integrity: sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==} - dev: false /mkdirp@0.5.6: resolution: {integrity: sha512-FP+p8RB8OWpF3YZBCrP5gtADmtXApB5AMLn+vdyA+PyxCjrCs00mjyUozssO33cwDeT3wNGdLxJ5M//YqtHAJw==} @@ -21490,7 +21482,6 @@ packages: /napi-build-utils@2.0.0: resolution: {integrity: sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==} - dev: false /napi-postinstall@0.3.4: resolution: {integrity: sha512-PHI5f1O0EP5xJ9gQmFGMS6IZcrVvTjpXjz7Na41gTE7eE2hK11lg04CECCYEEjdc17EV4DO+fkGEtt7TpTaTiQ==} @@ -21533,7 +21524,6 @@ packages: engines: {node: '>=10'} dependencies: semver: 7.7.3 - dev: false /node-abort-controller@3.1.1: resolution: {integrity: sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ==} @@ -22598,7 +22588,6 @@ packages: simple-get: 4.0.1 tar-fs: 2.1.4 tunnel-agent: 0.6.0 - dev: false /prelude-ls@1.2.1: resolution: {integrity: sha512-vkcDPrRZo1QZLbn5RLGPpg/WmIQ65qoWWhcGKf/b5eplkkarX0m9z8ppCat4mlOqUsWpyNuYgO3VRyrYHSzX5g==} @@ -22740,7 +22729,6 @@ packages: dependencies: end-of-stream: 1.4.5 once: 1.4.0 - dev: false /punycode.js@2.3.1: resolution: {integrity: sha512-uxFIHU0YlHYhDQtV4R9J6a52SLx28BCjT+4ieh7IGbgwVJWO+km431c4yRlREUAsAmt/uMjQUyQHNEPf0M39CA==} @@ -22826,7 +22814,6 @@ packages: ini: 1.3.8 minimist: 1.2.8 strip-json-comments: 2.0.1 - dev: false /react-devtools-core@5.3.2: resolution: {integrity: sha512-crr9HkVrDiJ0A4zot89oS0Cgv0Oa4OG1Em4jit3P3ZxZSKPMYyMjfwMqgcJna9o625g8oN87rBm8SWWrSTBZxg==} @@ -24138,7 +24125,6 @@ packages: /simple-concat@1.0.1: resolution: {integrity: sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==} - dev: false /simple-get@4.0.1: resolution: {integrity: sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==} @@ -24146,7 +24132,6 @@ packages: decompress-response: 6.0.0 once: 1.4.0 simple-concat: 1.0.1 - dev: false /simple-git@3.30.0: resolution: {integrity: sha512-q6lxyDsCmEal/MEGhP1aVyQ3oxnagGlBDOVSIB4XUVLl1iZh0Pah6ebC9V4xBap/RfgP2WlI8EKs0WS0rMEJHg==} @@ -24630,7 +24615,6 @@ packages: /strip-json-comments@2.0.1: resolution: {integrity: sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==} engines: {node: '>=0.10.0'} - dev: false /strip-json-comments@3.1.1: resolution: {integrity: sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig==} @@ -24838,7 +24822,6 @@ packages: mkdirp-classic: 0.5.3 pump: 3.0.3 tar-stream: 2.2.0 - dev: false /tar-stream@2.2.0: resolution: {integrity: sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==} @@ -24849,7 +24832,6 @@ packages: fs-constants: 1.0.0 inherits: 2.0.4 readable-stream: 3.6.2 - dev: false /tar@6.2.1: resolution: {integrity: sha512-DZ4yORTwrbTj/7MZYq2w+/ZFdI6OZ/f9SFHR+71gIVUZhOQPHzVCLpvRnPgyaMpfWxxk/4ONva3GQSyNIKRv6A==} @@ -25492,7 +25474,6 @@ packages: resolution: {integrity: sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==} dependencies: safe-buffer: 5.2.1 - dev: false /turbo-darwin-64@2.7.2: resolution: {integrity: sha512-dxY3X6ezcT5vm3coK6VGixbrhplbQMwgNsCsvZamS/+/6JiebqW9DKt4NwpgYXhDY2HdH00I7FWs3wkVuan4rA==} @@ -25649,6 +25630,83 @@ packages: /typedarray@0.0.6: resolution: {integrity: sha512-/aCDEGatGvZ2BIk+HmLf4ifCJFwvKFNb9/JeZPMulfgFracn9QFcAf5GO8B/mweUjSoblS5In0cWhqpfs/5PQA==} + /typeorm@0.3.28(better-sqlite3@11.10.0)(pg@8.16.3): + resolution: {integrity: sha512-6GH7wXhtfq2D33ZuRXYwIsl/qM5685WZcODZb7noOOcRMteM9KF2x2ap3H0EBjnSV0VO4gNAfJT5Ukp0PkOlvg==} + engines: {node: '>=16.13.0'} + hasBin: true + peerDependencies: + '@google-cloud/spanner': ^5.18.0 || ^6.0.0 || ^7.0.0 || ^8.0.0 + '@sap/hana-client': ^2.14.22 + better-sqlite3: ^8.0.0 || ^9.0.0 || ^10.0.0 || ^11.0.0 || ^12.0.0 + ioredis: ^5.0.4 + mongodb: ^5.8.0 || ^6.0.0 + mssql: ^9.1.1 || ^10.0.0 || ^11.0.0 || ^12.0.0 + mysql2: ^2.2.5 || ^3.0.1 + oracledb: ^6.3.0 + pg: ^8.5.1 + pg-native: ^3.0.0 + pg-query-stream: ^4.0.0 + redis: ^3.1.1 || ^4.0.0 || ^5.0.14 + sql.js: ^1.4.0 + sqlite3: ^5.0.3 + ts-node: ^10.7.0 + typeorm-aurora-data-api-driver: ^2.0.0 || ^3.0.0 + peerDependenciesMeta: + '@google-cloud/spanner': + optional: true + '@sap/hana-client': + optional: true + better-sqlite3: + optional: true + ioredis: + optional: true + mongodb: + optional: true + mssql: + optional: true + mysql2: + optional: true + oracledb: + optional: true + pg: + optional: true + pg-native: + optional: true + pg-query-stream: + optional: true + redis: + optional: true + sql.js: + optional: true + sqlite3: + optional: true + ts-node: + optional: true + typeorm-aurora-data-api-driver: + optional: true + dependencies: + '@sqltools/formatter': 1.2.5 + ansis: 4.2.0 + app-root-path: 3.1.0 + better-sqlite3: 11.10.0 + buffer: 6.0.3 + dayjs: 1.11.19 + debug: 4.4.3 + dedent: 1.7.1 + dotenv: 16.6.1 + glob: 10.5.0 + pg: 8.16.3 + reflect-metadata: 0.2.2 + sha.js: 2.4.12 + sql-highlight: 6.1.0 + tslib: 2.8.1 + uuid: 11.1.0 + yargs: 17.7.2 + transitivePeerDependencies: + - babel-plugin-macros + - supports-color + dev: false + /typeorm@0.3.28(better-sqlite3@11.10.0)(ts-node@10.9.2): resolution: {integrity: sha512-6GH7wXhtfq2D33ZuRXYwIsl/qM5685WZcODZb7noOOcRMteM9KF2x2ap3H0EBjnSV0VO4gNAfJT5Ukp0PkOlvg==} engines: {node: '>=16.13.0'}