From eb0b87315ebffe35a04e3024c116caf2078934f1 Mon Sep 17 00:00:00 2001 From: "Mr.doob" Date: Thu, 20 Mar 2025 15:52:43 +0900 Subject: [PATCH 01/12] Editor: Added agent. --- editor/css/main.css | 70 ++++ editor/index.html | 7 +- editor/js/Agent.js | 833 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 909 insertions(+), 1 deletion(-) create mode 100644 editor/js/Agent.js diff --git a/editor/css/main.css b/editor/css/main.css index f07406cea30262..8c71cc33cdffe7 100644 --- a/editor/css/main.css +++ b/editor/css/main.css @@ -792,3 +792,73 @@ select { transform: translate(0, 0); z-index: 0; } + +/* Agent */ +#agent { + position: absolute; + left: 50%; + bottom: 20px; + transform: translateX(-50%); + width: 400px; + box-sizing: border-box; + background-color: #ffffff; + padding: 12px; + border-radius: 12px; + color: #333333; + font-family: Arial, sans-serif; + z-index: 1000; + box-shadow: 0 2px 12px rgba(0,0,0,0.1); +} + +#agent textarea { + width: 100%; + height: 50px; + padding: 15px; + padding-right: 85px; + background-color: #f5f5f5; + border: 1px solid #e0e0e0; + border-radius: 8px; + color: #333333; + resize: none; + font-size: 14px; + box-sizing: border-box; +} + +#agent button { + position: absolute; + right: 25px; + bottom: 25px; + background-color: #2196F3; + color: white; + padding: 8px 16px; + border: none; + border-radius: 6px; + cursor: pointer; + font-weight: bold; + font-size: 13px; + transition: background-color 0.2s, opacity 0.2s; +} + +#agent button:hover:not(:disabled) { + background-color: #1976D2; +} + +#agent button:disabled { + opacity: 0.5; + cursor: default; +} + +/* Dark mode support */ +@media (prefers-color-scheme: dark) { + #agent { + background-color: #222; + color: #eee; + box-shadow: 0 2px 12px rgba(0,0,0,0.3); + } + + #agent textarea { + background-color: #333; + border-color: #444; + color: #eee; + } +} diff --git a/editor/index.html b/editor/index.html index 21768496d07ee1..225efa0b8750f7 100644 --- a/editor/index.html +++ b/editor/index.html @@ -20,7 +20,8 @@ "three/examples/": "../examples/", "three-gpu-pathtracer": "https://cdn.jsdelivr.net/npm/three-gpu-pathtracer@0.0.23/build/index.module.js", - "three-mesh-bvh": "https://cdn.jsdelivr.net/npm/three-mesh-bvh@0.7.4/build/index.module.js" + "three-mesh-bvh": "https://cdn.jsdelivr.net/npm/three-mesh-bvh@0.7.4/build/index.module.js", + "@google/genai": "https://cdn.jsdelivr.net/npm/@google/genai/dist/web/index.mjs" } } @@ -70,6 +71,7 @@ import { Sidebar } from './js/Sidebar.js'; import { Menubar } from './js/Menubar.js'; import { Resizer } from './js/Resizer.js'; + import { Agent } from './js/Agent.js'; window.URL = window.URL || window.webkitURL; window.BlobBuilder = window.BlobBuilder || window.WebKitBlobBuilder || window.MozBlobBuilder; @@ -102,6 +104,9 @@ const resizer = new Resizer( editor ); document.body.appendChild( resizer.dom ); + const agent = new Agent( editor ); + viewport.dom.appendChild( agent.dom ); + // editor.storage.init( function () { diff --git a/editor/js/Agent.js b/editor/js/Agent.js new file mode 100644 index 00000000000000..5b4aaef4825277 --- /dev/null +++ b/editor/js/Agent.js @@ -0,0 +1,833 @@ +import { GoogleGenAI } from '@google/genai'; +import * as Commands from './commands/Commands.js'; +import { Vector3, BoxGeometry, SphereGeometry, MeshStandardMaterial, Mesh, DirectionalLight, PointLight, AmbientLight, Color, CylinderGeometry } from 'three'; + +class Agent { + + constructor( editor ) { + + this.editor = editor; + this.container = new THREE.Group(); + this.dom = document.createElement( 'div' ); + this.dom.id = 'agent'; + this.lastModifiedObject = null; // Track last modified object + + // Create UI elements + this.createUI(); + + // Initialize signals + this.signals = { + agentResponse: new signals.Signal(), + agentThinking: new signals.Signal() + }; + + // Bind methods + this.processQuery = this.processQuery.bind( this ); + this.executeCommand = this.executeCommand.bind( this ); + this.generateRandomColor = this.generateRandomColor.bind( this ); + this.generateUniqueObjectName = this.generateUniqueObjectName.bind( this ); + + } + + generateUniqueObjectName( baseName ) { + + const scene = this.editor.scene; + let counter = 1; + let name; + + // Keep incrementing counter until we find an unused name + do { + + name = `${baseName}${counter}`; + counter ++; + + } while ( scene.getObjectByName( name ) !== undefined ); + + return name; + + } + + generateRandomColor() { + + const randomHex = Math.floor( Math.random() * 16777215 ).toString( 16 ); + return '#' + randomHex.padStart( 6, '0' ); + + } + + createUI() { + + // Create input area + const input = document.createElement( 'textarea' ); + input.placeholder = 'What do you want to do?'; + + // Prevent keyboard shortcuts when focused + input.addEventListener( 'keydown', ( e ) => { + + e.stopPropagation(); + + if ( e.key === 'Enter' ) { + + if ( e.shiftKey ) { + + // Allow Shift+Enter for newlines + return; + + } + + e.preventDefault(); + executeQuery(); + + } + + } ); + + // Create submit button + const button = document.createElement( 'button' ); + button.textContent = 'SEND'; + + const executeQuery = async () => { + + if ( button.disabled || ! input.value.trim() ) return; + + button.disabled = true; + input.disabled = true; + + await this.processQuery( input.value ); + + input.value = ''; + button.disabled = false; + input.disabled = false; + input.focus(); + + }; + + // Add event listeners + button.addEventListener( 'click', executeQuery ); + + // Append elements + this.dom.appendChild( input ); + this.dom.appendChild( button ); + + } + + async processQuery( query ) { + + if ( ! query.trim() ) return; + + try { + + this.signals.agentThinking.dispatch(); + + // Initialize Google AI + const ai = new GoogleGenAI( { apiKey: 'GEMINI_API_KEY' } ); + + // Get scene information + const sceneInfo = this.getSceneInfo(); + + // Prepare prompt + const prompt = `You are a Three.js scene manipulation assistant. Current scene info: + ${JSON.stringify( sceneInfo, null, 2 )} + + Available commands: + - AddObject: Add a new object to the scene + Types: box/cube, sphere, directionalLight, pointLight, ambientLight, cylinder + Box parameters: + - width, height, depth (default: 1) + - widthSegments, heightSegments, depthSegments (default: 1) - controls geometry detail + Sphere parameters: + - radius (default: 0.5) + - widthSegments (default: 32) - horizontal detail + - heightSegments (default: 16) - vertical detail + Cylinder parameters: + - radiusTop (default: 0.5) + - radiusBottom (default: 0.5) + - height (default: 1) + - radialSegments (default: 32) - horizontal detail + - heightSegments (default: 1) - vertical detail + - openEnded (default: false) + DirectionalLight parameters: + - color (default: white) + - intensity (default: 1) + PointLight parameters: + - color (default: white) + - intensity (default: 1) + - distance (default: 0) + - decay (default: 2) + AmbientLight parameters: + - color (default: white) + - intensity (default: 1) + Common parameters for all: + - color (use simple color names like "red" or hex values like "#ff0000" - do not use functions or dynamic values) + - position (e.g. {x: 0, y: 5, z: 0}) + - SetPosition: Set object position + Parameters: + - object: name of the object to move (optional - defaults to last modified object) + - position: {x, y, z} (omitted coordinates keep current values) + Example: Move right = {x: 2}, Move up = {y: 2} + - SetMaterialColor: Change object material color + Parameters: + - object: name of the object (optional - defaults to last modified object) + - color: color value (e.g. "red", "#ff0000", or "random" for a random color) + Note: Use "random" keyword for random colors, do not use JavaScript expressions + - SetScale: Change object size + Parameters: + - object: name of the object (optional - defaults to last modified object) + - scale: {x, y, z} (values > 1 make bigger, < 1 make smaller) + Example: Double size = {x: 2, y: 2, z: 2} + Example: Half size = {x: 0.5, y: 0.5, z: 0.5} + - SetMaterialValue: Set material property value + Parameters: + - object: name of the object (optional - defaults to last modified object) + - property: material property to set (e.g. "wireframe") + - value: value to set + - SetRotation: Set object rotation + Parameters: + - object: name of the object (optional - defaults to last modified object) + - rotation: {x, y, z} in radians + - SetGeometry: Modify object geometry detail + Parameters: + - object: name of the object to modify (optional - defaults to last modified object) + - widthSegments: number of segments along width (for box/sphere) + - heightSegments: number of segments along height (for box/sphere) + - depthSegments: number of segments along depth (for box only) + Example: High detail sphere = { widthSegments: 64, heightSegments: 32 } + Example: High detail box = { widthSegments: 4, heightSegments: 4, depthSegments: 4 } + - RemoveObject: Remove an object from the scene + Parameters: + - object: name of the object to remove + - MultiCmds: Execute multiple commands in sequence + Parameters: + - commands: array of command objects + Example - Create multiple objects: + { + "type": "MultiCmds", + "params": { + "commands": [ + { + "type": "AddObject", + "params": { + "type": "cube", + "name": "Cube1", + "position": {"x": -1.5} + } + }, + { + "type": "AddObject", + "params": { + "type": "cube", + "name": "Cube2", + "position": {"x": -0.5} + } + }, + { + "type": "AddObject", + "params": { + "type": "cube", + "name": "Cube3", + "position": {"x": 0.5} + } + }, + { + "type": "AddObject", + "params": { + "type": "cube", + "name": "Cube4", + "position": {"x": 1.5} + } + } + ] + } + } + Example - Create and modify an object: + { + "type": "MultiCmds", + "params": { + "commands": [ + { + "type": "AddObject", + "params": { "type": "cube", "name": "MyCube" } + }, + { + "type": "SetMaterialColor", + "params": { "object": "MyCube", "color": "red" } + }, + { + "type": "SetScale", + "params": { "object": "MyCube", "scale": {"x": 2, "y": 2, "z": 2} } + } + ] + } + } + Example - Modify all objects in the scene: + { + "type": "MultiCmds", + "params": { + "commands": [ + { + "type": "SetMaterialColor", + "params": { "object": "Box1", "color": "red" } + }, + { + "type": "SetMaterialColor", + "params": { "object": "Box2", "color": "blue" } + } + ] + } + } + Note: Use MultiCmds when you need to: + 1. Create multiple objects at once + 2. Apply multiple modifications to a single object + 3. Apply modifications to multiple objects + 4. Any combination of the above + + Important: When working with multiple similar objects (e.g. multiple spheres): + - Objects are automatically numbered (e.g. "Sphere1", "Sphere2", etc.) + - Use the exact object name including the number when targeting specific objects + - To modify all objects of a type, create a MultiCmds command with one command per object + - The scene info includes: + - objectCounts: how many of each type exist + - objectsByType: groups of objects by their base name + - spheres: list of all sphere names + - boxes: list of all box names + - cylinders: list of all cylinder names + - directionalLights: list of all directional light names + - pointLights: list of all point light names + - ambientLights: list of all ambient light names + + Example - Set random colors for all spheres: + { + "type": "MultiCmds", + "params": { + "commands": [ + { + "type": "SetMaterialColor", + "params": { "object": "Sphere1", "color": "random" } + }, + { + "type": "SetMaterialColor", + "params": { "object": "Sphere2", "color": "random" } + } + ] + } + } + + User query: ${query} + + Respond ONLY with a JSON object in this format: + { + "response": "Your text response to the user explaining what you're doing", + "commands": { + "type": "command_type", + "params": { + // command specific parameters + } + } + } + + Important: + 1. If no commands are needed, set "commands" to null + 2. Do not include any JavaScript expressions or functions in the JSON + 3. For random colors, use the "random" keyword instead of Math.random() + 4. Do not include any other text outside the JSON + + Do not include any other text outside the JSON.`; + + // Get response + const response = await ai.models.generateContent( { + model: 'gemini-2.0-flash-001', + contents: prompt, + generationConfig: { + temperature: 0.1, // Lower temperature for more consistent JSON output + maxOutputTokens: 2048 + } + } ); + + let responseData; + + try { + + // Strip markdown code block markers if present + const cleanText = response.text.replace( /^```json\n|\n```$/g, '' ) + .replace( /^\s*```\s*|\s*```\s*$/g, '' ) // Remove any remaining code block markers + .trim(); + + try { + + // First try parsing as is + responseData = JSON.parse( cleanText ); + + } catch ( e ) { + + // If that fails, try to fix common JSON issues + const fixedText = cleanText + .replace( /,\s*([}\]])/g, '$1' ) // Remove trailing commas + .replace( /([a-zA-Z0-9])\s*:\s*/g, '"$1": ' ) // Quote unquoted keys + .replace( /\n/g, ' ' ) // Remove newlines + .replace( /\s+/g, ' ' ); // Normalize whitespace + + responseData = JSON.parse( fixedText ); + + } + + } catch ( e ) { + + console.error( 'AGENT: Failed to parse AI response as JSON:', e ); + console.error( 'AGENT: Raw response:', response.text ); + return; + + } + + // Execute commands if present + if ( responseData.commands ) { + + try { + + await this.executeCommand( responseData.commands ); + + } catch ( e ) { + + console.error( 'AGENT: Failed to execute commands:', e ); + + } + + } + + // Log the response + console.log( 'AGENT:', responseData.response ); + this.signals.agentResponse.dispatch( responseData.response ); + + } catch ( error ) { + + console.error( 'AGENT: Agent error:', error ); + + } + + } + + async executeCommand( commandData ) { + + if ( ! commandData.type || ! Commands[ commandData.type + 'Command' ] ) { + + console.error( 'AGENT: Invalid command type:', commandData.type ); + return; + + } + + let command; + + // Helper to get target object, falling back to last modified + const getTargetObject = ( objectName ) => { + + if ( objectName ) { + + const object = this.editor.scene.getObjectByName( objectName ); + if ( object ) { + + this.lastModifiedObject = object; + return object; + + } + + } + + return this.lastModifiedObject; + + }; + + const createMaterial = ( params ) => { + + const material = new MeshStandardMaterial(); + + if ( params.color ) { + + material.color.set( params.color ); + + } + + return material; + + }; + + const setPosition = ( object, position ) => { + + if ( position ) { + + object.position.set( + position.x ?? 0, + position.y ?? 0, + position.z ?? 0 + ); + + } + + }; + + switch ( commandData.type ) { + + case 'AddObject': + + const type = commandData.params.type?.toLowerCase(); + + if ( type === 'box' || type === 'cube' ) { + + const width = commandData.params.width ?? 1; + const height = commandData.params.height ?? 1; + const depth = commandData.params.depth ?? 1; + const widthSegments = commandData.params.widthSegments ?? 1; + const heightSegments = commandData.params.heightSegments ?? 1; + const depthSegments = commandData.params.depthSegments ?? 1; + const geometry = new BoxGeometry( width, height, depth, widthSegments, heightSegments, depthSegments ); + const mesh = new Mesh( geometry, createMaterial( commandData.params ) ); + mesh.name = commandData.params.name || this.generateUniqueObjectName( 'Box' ); + + setPosition( mesh, commandData.params.position ); + + command = new Commands.AddObjectCommand( this.editor, mesh ); + this.lastModifiedObject = mesh; + + } else if ( type === 'sphere' ) { + + const radius = commandData.params.radius ?? 0.5; + const widthSegments = commandData.params.widthSegments ?? 32; + const heightSegments = commandData.params.heightSegments ?? 16; + const geometry = new SphereGeometry( radius, widthSegments, heightSegments ); + const mesh = new Mesh( geometry, createMaterial( commandData.params ) ); + mesh.name = commandData.params.name || this.generateUniqueObjectName( 'Sphere' ); + + setPosition( mesh, commandData.params.position ); + + command = new Commands.AddObjectCommand( this.editor, mesh ); + this.lastModifiedObject = mesh; + + } else if ( type === 'cylinder' ) { + + const radiusTop = commandData.params.radiusTop ?? 0.5; + const radiusBottom = commandData.params.radiusBottom ?? 0.5; + const height = commandData.params.height ?? 1; + const radialSegments = commandData.params.radialSegments ?? 32; + const heightSegments = commandData.params.heightSegments ?? 1; + const openEnded = commandData.params.openEnded ?? false; + const geometry = new CylinderGeometry( radiusTop, radiusBottom, height, radialSegments, heightSegments, openEnded ); + const mesh = new Mesh( geometry, createMaterial( commandData.params ) ); + mesh.name = commandData.params.name || this.generateUniqueObjectName( 'Cylinder' ); + + setPosition( mesh, commandData.params.position ); + + command = new Commands.AddObjectCommand( this.editor, mesh ); + this.lastModifiedObject = mesh; + + } else if ( type === 'directionallight' ) { + + const color = commandData.params.color || 0xffffff; + const intensity = commandData.params.intensity ?? 1; + const light = new DirectionalLight( color, intensity ); + light.name = commandData.params.name || this.generateUniqueObjectName( 'DirectionalLight' ); + + setPosition( light, commandData.params.position ); + + command = new Commands.AddObjectCommand( this.editor, light ); + this.lastModifiedObject = light; + + } else if ( type === 'pointlight' ) { + + const color = commandData.params.color || 0xffffff; + const intensity = commandData.params.intensity ?? 1; + const distance = commandData.params.distance ?? 0; + const decay = commandData.params.decay ?? 2; + const light = new PointLight( color, intensity, distance, decay ); + light.name = commandData.params.name || this.generateUniqueObjectName( 'PointLight' ); + + setPosition( light, commandData.params.position ); + + command = new Commands.AddObjectCommand( this.editor, light ); + this.lastModifiedObject = light; + + } else if ( type === 'ambientlight' ) { + + const color = commandData.params.color || 0xffffff; + const intensity = commandData.params.intensity ?? 1; + const light = new AmbientLight( color, intensity ); + light.name = commandData.params.name || this.generateUniqueObjectName( 'AmbientLight' ); + + command = new Commands.AddObjectCommand( this.editor, light ); + this.lastModifiedObject = light; + + } else { + + console.warn( 'AGENT: Unsupported object type:', type ); + + } + + break; + + case 'SetPosition': + + const positionObject = getTargetObject( commandData.params.object ); + + if ( positionObject && commandData.params.position ) { + + const currentPos = positionObject.position; + const newPosition = new Vector3( + commandData.params.position.x ?? currentPos.x, + commandData.params.position.y ?? currentPos.y, + commandData.params.position.z ?? currentPos.z + ); + command = new Commands.SetPositionCommand( this.editor, positionObject, newPosition ); + + } + + break; + + case 'SetRotation': + + const rotationObject = getTargetObject( commandData.params.object ); + + if ( rotationObject && commandData.params.rotation ) { + + const rot = commandData.params.rotation; + const currentRot = rotationObject.rotation; + const newRotation = new Vector3( + rot.x ?? currentRot.x, + rot.y ?? currentRot.y, + rot.z ?? currentRot.z + ); + command = new Commands.SetRotationCommand( this.editor, rotationObject, newRotation ); + + } + + break; + + case 'SetScale': + + const scaleObject = getTargetObject( commandData.params.object ); + + if ( scaleObject && commandData.params.scale ) { + + const scale = commandData.params.scale; + const newScale = new Vector3( scale.x || 1, scale.y || 1, scale.z || 1 ); + command = new Commands.SetScaleCommand( this.editor, scaleObject, newScale ); + + } + + break; + + case 'SetMaterialColor': + + const colorObject = getTargetObject( commandData.params.object ); + + if ( colorObject && colorObject.material && commandData.params.color ) { + + let colorValue = commandData.params.color; + // If color is "random", generate a random color + if ( colorValue === 'random' ) { + + colorValue = this.generateRandomColor(); + + } + + const color = new Color( colorValue ); + command = new Commands.SetMaterialColorCommand( this.editor, colorObject, 'color', color.getHex() ); + + } + + break; + + case 'SetMaterialValue': + + const materialObject = getTargetObject( commandData.params.object ); + + if ( materialObject && materialObject.material && commandData.params.property ) { + + const value = commandData.params.value ?? true; + command = new Commands.SetMaterialValueCommand( this.editor, materialObject, commandData.params.property, value ); + + } + + break; + + case 'SetGeometry': + + const detailObject = getTargetObject( commandData.params.object ); + + if ( detailObject && detailObject.geometry ) { + + const params = commandData.params; + let newGeometry; + + if ( detailObject.geometry instanceof BoxGeometry ) { + + const box = detailObject.geometry; + newGeometry = new BoxGeometry( + box.parameters.width ?? 1, + box.parameters.height ?? 1, + box.parameters.depth ?? 1, + params.widthSegments ?? 1, + params.heightSegments ?? 1, + params.depthSegments ?? 1 + ); + + } else if ( detailObject.geometry instanceof SphereGeometry ) { + + const sphere = detailObject.geometry; + newGeometry = new SphereGeometry( + sphere.parameters.radius ?? 0.5, + params.widthSegments ?? 32, + params.heightSegments ?? 16 + ); + + } else if ( detailObject.geometry instanceof CylinderGeometry ) { + + const cylinder = detailObject.geometry; + newGeometry = new CylinderGeometry( + params.radiusTop ?? cylinder.parameters.radiusTop ?? 0.5, + params.radiusBottom ?? cylinder.parameters.radiusBottom ?? 0.5, + params.height ?? cylinder.parameters.height ?? 1, + params.radialSegments ?? cylinder.parameters.radialSegments ?? 32, + params.heightSegments ?? cylinder.parameters.heightSegments ?? 1, + params.openEnded ?? cylinder.parameters.openEnded ?? false + ); + + } + + if ( newGeometry ) { + + command = new Commands.SetGeometryCommand( this.editor, detailObject, newGeometry ); + + } + + } + + break; + + case 'RemoveObject': + + const removeObject = getTargetObject( commandData.params.object ); + + if ( removeObject ) { + + command = new Commands.RemoveObjectCommand( this.editor, removeObject ); + this.lastModifiedObject = null; + + } + + break; + + case 'MultiCmds': + + if ( Array.isArray( commandData.params.commands ) ) { + + const commands = []; + + for ( const cmd of commandData.params.commands ) { + + const subCommand = await this.executeCommand( cmd ); + if ( subCommand ) commands.push( subCommand ); + + } + + command = new Commands.MultiCmdsCommand( this.editor, commands ); + + } + + break; + + default: + console.warn( 'AGENT: Unsupported command type:', commandData.type, '- Available commands are: AddObject, SetPosition, SetRotation, SetScale, SetMaterialColor, SetMaterialValue, SetGeometry, RemoveObject, MultiCmds' ); + break; + + } + + console.log( 'AGENT: Command:', command ); + + if ( command ) { + + this.editor.execute( command ); + + } + + return command; + + } + + getSceneInfo() { + + const scene = this.editor.scene; + + // Helper to get all objects of a specific type + const getObjectsByType = ( type ) => { + + return scene.children.filter( obj => { + + const baseName = obj.name.replace( /\d+$/, '' ); + return baseName.toLowerCase() === type.toLowerCase(); + + } ).map( obj => obj.name ); + + }; + + // Get base names and their counts + const nameCount = {}; + const objectsByType = {}; + + scene.children.forEach( obj => { + + const baseName = obj.name.replace( /\d+$/, '' ); // Remove trailing numbers + nameCount[ baseName ] = ( nameCount[ baseName ] || 0 ) + 1; + + // Group objects by their base name + if ( ! objectsByType[ baseName ] ) { + + objectsByType[ baseName ] = []; + + } + + objectsByType[ baseName ].push( obj.name ); + + } ); + + const objects = scene.children.map( obj => ( { + type: obj.type, + name: obj.name, + baseName: obj.name.replace( /\d+$/, '' ), // Add base name + position: obj.position, + rotation: obj.rotation, + scale: obj.scale, + isMesh: obj.isMesh, + isLight: obj.isLight, + material: obj.material ? { + type: obj.material.type, + color: obj.material.color ? '#' + obj.material.color.getHexString() : undefined + } : undefined + } ) ); + + return { + objects, + meshes: objects.filter( obj => obj.isMesh ), + lights: objects.filter( obj => obj.isLight ), + materials: Object.keys( this.editor.materials ).length, + cameras: Object.keys( this.editor.cameras ).length, + objectCounts: nameCount, // Add counts of similar objects + objectsByType, // Add grouped objects by type + spheres: getObjectsByType( 'Sphere' ), + boxes: getObjectsByType( 'Box' ), + cylinders: getObjectsByType( 'Cylinder' ), + directionalLights: getObjectsByType( 'DirectionalLight' ), + pointLights: getObjectsByType( 'PointLight' ), + ambientLights: getObjectsByType( 'AmbientLight' ) + }; + + } + + clear() { + + while ( this.container.children.length > 0 ) { + + this.container.remove( this.container.children[ 0 ] ); + + } + + } + +} + +export { Agent }; From 2e0f12506994f709cbd654711b185e3d03f3cb2a Mon Sep 17 00:00:00 2001 From: "Mr.doob" Date: Fri, 21 Mar 2025 13:52:17 +0900 Subject: [PATCH 02/12] Now using ai.chats.create() --- editor/js/Agent.js | 457 ++++++++++++++++++++++++--------------------- 1 file changed, 239 insertions(+), 218 deletions(-) diff --git a/editor/js/Agent.js b/editor/js/Agent.js index 5b4aaef4825277..6dbbb03ea6fd0a 100644 --- a/editor/js/Agent.js +++ b/editor/js/Agent.js @@ -27,6 +27,8 @@ class Agent { this.generateRandomColor = this.generateRandomColor.bind( this ); this.generateUniqueObjectName = this.generateUniqueObjectName.bind( this ); + this.init(); + } generateUniqueObjectName( baseName ) { @@ -110,237 +112,256 @@ class Agent { } - async processQuery( query ) { - - if ( ! query.trim() ) return; - - try { + async init() { - this.signals.agentThinking.dispatch(); - - // Initialize Google AI + // Initialize Google AI const ai = new GoogleGenAI( { apiKey: 'GEMINI_API_KEY' } ); - // Get scene information - const sceneInfo = this.getSceneInfo(); - - // Prepare prompt - const prompt = `You are a Three.js scene manipulation assistant. Current scene info: - ${JSON.stringify( sceneInfo, null, 2 )} - - Available commands: - - AddObject: Add a new object to the scene - Types: box/cube, sphere, directionalLight, pointLight, ambientLight, cylinder - Box parameters: - - width, height, depth (default: 1) - - widthSegments, heightSegments, depthSegments (default: 1) - controls geometry detail - Sphere parameters: - - radius (default: 0.5) - - widthSegments (default: 32) - horizontal detail - - heightSegments (default: 16) - vertical detail - Cylinder parameters: - - radiusTop (default: 0.5) - - radiusBottom (default: 0.5) - - height (default: 1) - - radialSegments (default: 32) - horizontal detail - - heightSegments (default: 1) - vertical detail - - openEnded (default: false) - DirectionalLight parameters: - - color (default: white) - - intensity (default: 1) - PointLight parameters: - - color (default: white) - - intensity (default: 1) - - distance (default: 0) - - decay (default: 2) - AmbientLight parameters: - - color (default: white) - - intensity (default: 1) - Common parameters for all: - - color (use simple color names like "red" or hex values like "#ff0000" - do not use functions or dynamic values) - - position (e.g. {x: 0, y: 5, z: 0}) - - SetPosition: Set object position - Parameters: - - object: name of the object to move (optional - defaults to last modified object) - - position: {x, y, z} (omitted coordinates keep current values) - Example: Move right = {x: 2}, Move up = {y: 2} - - SetMaterialColor: Change object material color - Parameters: - - object: name of the object (optional - defaults to last modified object) - - color: color value (e.g. "red", "#ff0000", or "random" for a random color) - Note: Use "random" keyword for random colors, do not use JavaScript expressions - - SetScale: Change object size - Parameters: - - object: name of the object (optional - defaults to last modified object) - - scale: {x, y, z} (values > 1 make bigger, < 1 make smaller) - Example: Double size = {x: 2, y: 2, z: 2} - Example: Half size = {x: 0.5, y: 0.5, z: 0.5} - - SetMaterialValue: Set material property value - Parameters: - - object: name of the object (optional - defaults to last modified object) - - property: material property to set (e.g. "wireframe") - - value: value to set - - SetRotation: Set object rotation - Parameters: - - object: name of the object (optional - defaults to last modified object) - - rotation: {x, y, z} in radians - - SetGeometry: Modify object geometry detail - Parameters: - - object: name of the object to modify (optional - defaults to last modified object) - - widthSegments: number of segments along width (for box/sphere) - - heightSegments: number of segments along height (for box/sphere) - - depthSegments: number of segments along depth (for box only) - Example: High detail sphere = { widthSegments: 64, heightSegments: 32 } - Example: High detail box = { widthSegments: 4, heightSegments: 4, depthSegments: 4 } - - RemoveObject: Remove an object from the scene - Parameters: - - object: name of the object to remove - - MultiCmds: Execute multiple commands in sequence - Parameters: - - commands: array of command objects - Example - Create multiple objects: - { - "type": "MultiCmds", - "params": { - "commands": [ - { - "type": "AddObject", - "params": { - "type": "cube", - "name": "Cube1", - "position": {"x": -1.5} - } - }, - { - "type": "AddObject", - "params": { - "type": "cube", - "name": "Cube2", - "position": {"x": -0.5} - } - }, - { - "type": "AddObject", - "params": { - "type": "cube", - "name": "Cube3", - "position": {"x": 0.5} - } - }, - { - "type": "AddObject", - "params": { - "type": "cube", - "name": "Cube4", - "position": {"x": 1.5} - } + // Get scene information + const sceneInfo = this.getSceneInfo(); + + // Prepare prompt + const systemPrompt = `You are a Three.js scene manipulation assistant. Current scene info: + + ${JSON.stringify( sceneInfo, null, 2 )} + + Available commands: + - AddObject: Add a new object to the scene + Types: box/cube, sphere, directionalLight, pointLight, ambientLight, cylinder + Box parameters: + - width, height, depth (default: 1) + - widthSegments, heightSegments, depthSegments (default: 1) - controls geometry detail + Sphere parameters: + - radius (default: 0.5) + - widthSegments (default: 32) - horizontal detail + - heightSegments (default: 16) - vertical detail + Cylinder parameters: + - radiusTop (default: 0.5) + - radiusBottom (default: 0.5) + - height (default: 1) + - radialSegments (default: 32) - horizontal detail + - heightSegments (default: 1) - vertical detail + - openEnded (default: false) + DirectionalLight parameters: + - color (default: white) + - intensity (default: 1) + PointLight parameters: + - color (default: white) + - intensity (default: 1) + - distance (default: 0) + - decay (default: 2) + AmbientLight parameters: + - color (default: white) + - intensity (default: 1) + Common parameters for all: + - color (use simple color names like "red" or hex values like "#ff0000" - do not use functions or dynamic values) + - position (e.g. {x: 0, y: 5, z: 0}) + - SetPosition: Set object position + Parameters: + - object: name of the object to move (optional - defaults to last modified object) + - position: {x, y, z} (omitted coordinates keep current values) + Example: Move right = {x: 2}, Move up = {y: 2} + - SetMaterialColor: Change object material color + Parameters: + - object: name of the object (optional - defaults to last modified object) + - color: color value (e.g. "red", "#ff0000", or "random" for a random color) + Note: Use "random" keyword for random colors, do not use JavaScript expressions + - SetScale: Change object size + Parameters: + - object: name of the object (optional - defaults to last modified object) + - scale: {x, y, z} (values > 1 make bigger, < 1 make smaller) + Example: Double size = {x: 2, y: 2, z: 2} + Example: Half size = {x: 0.5, y: 0.5, z: 0.5} + - SetMaterialValue: Set material property value + Parameters: + - object: name of the object (optional - defaults to last modified object) + - property: material property to set (e.g. "wireframe") + - value: value to set + - SetRotation: Set object rotation + Parameters: + - object: name of the object (optional - defaults to last modified object) + - rotation: {x, y, z} in radians + - SetGeometry: Modify object geometry detail + Parameters: + - object: name of the object to modify (optional - defaults to last modified object) + - widthSegments: number of segments along width (for box/sphere) + - heightSegments: number of segments along height (for box/sphere) + - depthSegments: number of segments along depth (for box only) + Example: High detail sphere = { widthSegments: 64, heightSegments: 32 } + Example: High detail box = { widthSegments: 4, heightSegments: 4, depthSegments: 4 } + - RemoveObject: Remove an object from the scene + Parameters: + - object: name of the object to remove + - MultiCmds: Execute multiple commands in sequence + Parameters: + - commands: array of command objects + Example - Create multiple objects: + { + "type": "MultiCmds", + "params": { + "commands": [ + { + "type": "AddObject", + "params": { + "type": "cube", + "name": "Cube1", + "position": {"x": -1.5} } - ] - } - } - Example - Create and modify an object: - { - "type": "MultiCmds", - "params": { - "commands": [ - { - "type": "AddObject", - "params": { "type": "cube", "name": "MyCube" } - }, - { - "type": "SetMaterialColor", - "params": { "object": "MyCube", "color": "red" } - }, - { - "type": "SetScale", - "params": { "object": "MyCube", "scale": {"x": 2, "y": 2, "z": 2} } + }, + { + "type": "AddObject", + "params": { + "type": "cube", + "name": "Cube2", + "position": {"x": -0.5} } - ] - } - } - Example - Modify all objects in the scene: - { - "type": "MultiCmds", - "params": { - "commands": [ - { - "type": "SetMaterialColor", - "params": { "object": "Box1", "color": "red" } - }, - { - "type": "SetMaterialColor", - "params": { "object": "Box2", "color": "blue" } + }, + { + "type": "AddObject", + "params": { + "type": "cube", + "name": "Cube3", + "position": {"x": 0.5} } - ] - } - } - Note: Use MultiCmds when you need to: - 1. Create multiple objects at once - 2. Apply multiple modifications to a single object - 3. Apply modifications to multiple objects - 4. Any combination of the above - - Important: When working with multiple similar objects (e.g. multiple spheres): - - Objects are automatically numbered (e.g. "Sphere1", "Sphere2", etc.) - - Use the exact object name including the number when targeting specific objects - - To modify all objects of a type, create a MultiCmds command with one command per object - - The scene info includes: - - objectCounts: how many of each type exist - - objectsByType: groups of objects by their base name - - spheres: list of all sphere names - - boxes: list of all box names - - cylinders: list of all cylinder names - - directionalLights: list of all directional light names - - pointLights: list of all point light names - - ambientLights: list of all ambient light names - - Example - Set random colors for all spheres: - { - "type": "MultiCmds", - "params": { - "commands": [ - { - "type": "SetMaterialColor", - "params": { "object": "Sphere1", "color": "random" } - }, - { - "type": "SetMaterialColor", - "params": { "object": "Sphere2", "color": "random" } + }, + { + "type": "AddObject", + "params": { + "type": "cube", + "name": "Cube4", + "position": {"x": 1.5} } - ] - } + } + ] } - - User query: ${query} - - Respond ONLY with a JSON object in this format: - { - "response": "Your text response to the user explaining what you're doing", - "commands": { - "type": "command_type", + } + Example - Create and modify an object: + { + "type": "MultiCmds", "params": { - // command specific parameters + "commands": [ + { + "type": "AddObject", + "params": { "type": "cube", "name": "MyCube" } + }, + { + "type": "SetMaterialColor", + "params": { "object": "MyCube", "color": "red" } + }, + { + "type": "SetScale", + "params": { "object": "MyCube", "scale": {"x": 2, "y": 2, "z": 2} } + } + ] + } + } + Example - Modify all objects in the scene: + { + "type": "MultiCmds", + "params": { + "commands": [ + { + "type": "SetMaterialColor", + "params": { "object": "Box1", "color": "red" } + }, + { + "type": "SetMaterialColor", + "params": { "object": "Box2", "color": "blue" } + } + ] + } + } + Note: Use MultiCmds when you need to: + 1. Create multiple objects at once + 2. Apply multiple modifications to a single object + 3. Apply modifications to multiple objects + 4. Any combination of the above + + Important: When working with multiple similar objects (e.g. multiple spheres): + - Objects are automatically numbered (e.g. "Sphere1", "Sphere2", etc.) + - Use the exact object name including the number when targeting specific objects + - To modify all objects of a type, create a MultiCmds command with one command per object + - The scene info includes: + - objectCounts: how many of each type exist + - objectsByType: groups of objects by their base name + - spheres: list of all sphere names + - boxes: list of all box names + - cylinders: list of all cylinder names + - directionalLights: list of all directional light names + - pointLights: list of all point light names + - ambientLights: list of all ambient light names + + Example - Set random colors for all spheres: + { + "type": "MultiCmds", + "params": { + "commands": [ + { + "type": "SetMaterialColor", + "params": { "object": "Sphere1", "color": "random" } + }, + { + "type": "SetMaterialColor", + "params": { "object": "Sphere2", "color": "random" } + } + ] } } - } - Important: - 1. If no commands are needed, set "commands" to null - 2. Do not include any JavaScript expressions or functions in the JSON - 3. For random colors, use the "random" keyword instead of Math.random() - 4. Do not include any other text outside the JSON - - Do not include any other text outside the JSON.`; - - // Get response - const response = await ai.models.generateContent( { - model: 'gemini-2.0-flash-001', - contents: prompt, - generationConfig: { - temperature: 0.1, // Lower temperature for more consistent JSON output - maxOutputTokens: 2048 + Respond ONLY with a JSON object in this format: + { + "response": "Your text response to the user explaining what you're doing", + "commands": { + "type": "command_type", + "params": { + // command specific parameters } - } ); + } + } + + Important: + 1. If no commands are needed, set "commands" to null + 2. Do not include any JavaScript expressions or functions in the JSON + 3. For random colors, use the "random" keyword instead of Math.random() + 4. Do not include any other text outside the JSON + + Do not include any other text outside the JSON.`; + + this.chat = await ai.chats.create({ + model: "gemini-2.0-flash", + history: [ + { + role: "user", + parts: [{ text: systemPrompt }], + }, + { + role: "model", + parts: [ + { + text: "I'm ready to help you create and modify your 3D scene.", + }, + ], + }, + ], + config: { + temperature: 0.2, + maxOutputTokens: 2048 + }, + }); + + console.log( 'CHAT:', this.chat ); + + } + + async processQuery( query ) { + + if ( ! query.trim() ) return; + + try { + + this.signals.agentThinking.dispatch(); + + const response = await this.chat.sendMessage( { message: query } ); let responseData; From b4b3e0c86076ec4cdcd4641043b522d9329adfd5 Mon Sep 17 00:00:00 2001 From: "Mr.doob" Date: Fri, 21 Mar 2025 15:47:29 +0900 Subject: [PATCH 03/12] Editor: Wait for storage before initialising agent. --- editor/index.html | 2 ++ editor/js/Agent.js | 6 +++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/editor/index.html b/editor/index.html index 225efa0b8750f7..c9aabb1d744945 100644 --- a/editor/index.html +++ b/editor/index.html @@ -129,6 +129,8 @@ } + editor.signals.storageLoaded.dispatch(); + } ); // diff --git a/editor/js/Agent.js b/editor/js/Agent.js index 6dbbb03ea6fd0a..8d69021f3f9cec 100644 --- a/editor/js/Agent.js +++ b/editor/js/Agent.js @@ -27,7 +27,11 @@ class Agent { this.generateRandomColor = this.generateRandomColor.bind( this ); this.generateUniqueObjectName = this.generateUniqueObjectName.bind( this ); - this.init(); + this.editor.signals.storageLoaded.add( () => { + + this.init(); + + } ); } From 685a366e4c0c94e16c1cc6d41fa292bf274523ba Mon Sep 17 00:00:00 2001 From: "Mr.doob" Date: Fri, 21 Mar 2025 16:05:18 +0900 Subject: [PATCH 04/12] Editor: Added new signal. --- editor/js/Editor.js | 1 + 1 file changed, 1 insertion(+) diff --git a/editor/js/Editor.js b/editor/js/Editor.js index 2cd5ea313250ba..3ae8a65b5cfe21 100644 --- a/editor/js/Editor.js +++ b/editor/js/Editor.js @@ -36,6 +36,7 @@ function Editor() { // notifications editorCleared: new Signal(), + storageLoaded: new Signal(), savingStarted: new Signal(), savingFinished: new Signal(), From 54d08e58d9c1675cab061d508b9ba352e992d403 Mon Sep 17 00:00:00 2001 From: "Mr.doob" Date: Fri, 21 Mar 2025 16:22:20 +0900 Subject: [PATCH 05/12] Added message bubble. --- editor/js/Agent.js | 120 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 103 insertions(+), 17 deletions(-) diff --git a/editor/js/Agent.js b/editor/js/Agent.js index 8d69021f3f9cec..ef4a01820b2a63 100644 --- a/editor/js/Agent.js +++ b/editor/js/Agent.js @@ -62,6 +62,37 @@ class Agent { createUI() { + // Create message bubble + const messageBubble = document.createElement( 'div' ); + messageBubble.style.display = 'none'; + messageBubble.style.padding = '8px 12px'; + messageBubble.style.borderRadius = '4px'; + messageBubble.style.marginBottom = '8px'; + messageBubble.style.fontSize = '14px'; + messageBubble.style.position = 'relative'; + + // Add message container first + const messageContainer = document.createElement( 'div' ); + messageContainer.className = 'message-text'; + messageContainer.style.marginRight = '20px'; // Make space for the close button + messageContainer.style.whiteSpace = 'pre-wrap'; + messageBubble.appendChild( messageContainer ); + + // Add close button + const closeButton = document.createElement( 'div' ); + closeButton.innerHTML = '×'; + closeButton.style.position = 'absolute'; + closeButton.style.top = '8px'; + closeButton.style.right = '8px'; + closeButton.style.cursor = 'pointer'; + closeButton.style.fontSize = '18px'; + closeButton.style.lineHeight = '14px'; + closeButton.style.zIndex = '1'; + closeButton.onclick = () => { + messageBubble.style.display = 'none'; + }; + messageBubble.appendChild( closeButton ); + // Create input area const input = document.createElement( 'textarea' ); input.placeholder = 'What do you want to do?'; @@ -111,23 +142,51 @@ class Agent { button.addEventListener( 'click', executeQuery ); // Append elements + this.dom.appendChild( messageBubble ); this.dom.appendChild( input ); this.dom.appendChild( button ); + // Store references + this.messageBubble = messageBubble; + } + + showError( message ) { + this.showMessage( message, 'error' ); + } + + showMessage( message, type = 'normal' ) { + // Get the message container + const messageContainer = this.messageBubble.querySelector( '.message-text' ); + const closeButton = this.messageBubble.querySelector( 'div:last-child' ); + + // Set styles based on message type + if ( type === 'error' ) { + this.messageBubble.style.backgroundColor = '#ffebee'; + this.messageBubble.style.color = '#d32f2f'; + closeButton.style.color = '#d32f2f'; + } else { + this.messageBubble.style.backgroundColor = '#e8f5e9'; + this.messageBubble.style.color = '#2e7d32'; + closeButton.style.color = '#2e7d32'; + } + + // Update message text + messageContainer.textContent = message; + this.messageBubble.style.display = 'block'; } async init() { - // Initialize Google AI - const ai = new GoogleGenAI( { apiKey: 'GEMINI_API_KEY' } ); + // Initialize Google AI + const ai = new GoogleGenAI( { apiKey: 'AIzaSyCFpCy19tslP3VeSLAPUM-zQjz6Ka9g5no' } ); - // Get scene information - const sceneInfo = this.getSceneInfo(); + // Get scene information + const sceneInfo = this.getSceneInfo(); - // Prepare prompt + // Prepare prompt const systemPrompt = `You are a Three.js scene manipulation assistant. Current scene info: - ${JSON.stringify( sceneInfo, null, 2 )} + ${JSON.stringify( sceneInfo, null, 2 )} Available commands: - AddObject: Add a new object to the scene @@ -179,8 +238,13 @@ class Agent { - SetMaterialValue: Set material property value Parameters: - object: name of the object (optional - defaults to last modified object) - - property: material property to set (e.g. "wireframe") - - value: value to set + - property: material property to set (e.g. "metalness", "roughness", "wireframe", "transparent", "opacity") + - value: value to set (numbers between 0-1 for metalness/roughness/opacity, true/false for wireframe/transparent) + Example: Make metallic = { property: "metalness", value: 1.0 } + Example: Make rough = { property: "roughness", value: 1.0 } + Example: Make reflective = Use MultiCmds to set both metalness=1.0 and roughness=0.0 + Example: Make transparent = { property: "transparent", value: true, opacity: 0.5 } + Note: For reflective surfaces, combine metalness=1.0 with roughness=0.0 using MultiCmds - SetRotation: Set object rotation Parameters: - object: name of the object (optional - defaults to last modified object) @@ -367,6 +431,8 @@ class Agent { const response = await this.chat.sendMessage( { message: query } ); + console.log( 'RESPONSE:', response.text ); + let responseData; try { @@ -396,8 +462,9 @@ class Agent { } catch ( e ) { - console.error( 'AGENT: Failed to parse AI response as JSON:', e ); - console.error( 'AGENT: Raw response:', response.text ); + // console.error( 'AGENT: Failed to parse AI response as JSON:', e ); + // console.error( 'AGENT: Raw response:', response.text ); + this.showError( response.text ); return; } @@ -412,18 +479,22 @@ class Agent { } catch ( e ) { console.error( 'AGENT: Failed to execute commands:', e ); + this.showError( 'Failed to execute command: ' + e.message ); + return; } } // Log the response - console.log( 'AGENT:', responseData.response ); + // console.log( 'AGENT:', responseData.response ); this.signals.agentResponse.dispatch( responseData.response ); + this.showMessage( responseData.response ); } catch ( error ) { console.error( 'AGENT: Agent error:', error ); + this.showError( 'Agent error: ' + error.message ); } @@ -663,8 +734,23 @@ class Agent { if ( materialObject && materialObject.material && commandData.params.property ) { - const value = commandData.params.value ?? true; - command = new Commands.SetMaterialValueCommand( this.editor, materialObject, commandData.params.property, value ); + const property = commandData.params.property; + let value = commandData.params.value; + + // Handle special cases for certain property types + if ( property.includes( 'map' ) && value === null ) { + + // Handle removing textures + value = null; + + } else if ( typeof value === 'string' && !isNaN( value ) ) { + + // Convert numeric strings to numbers + value = parseFloat( value ); + + } + + command = new Commands.SetMaterialValueCommand( this.editor, materialObject, property, value ); } @@ -811,11 +897,11 @@ class Agent { } ); const objects = scene.children.map( obj => ( { - type: obj.type, - name: obj.name, + type: obj.type, + name: obj.name, baseName: obj.name.replace( /\d+$/, '' ), // Add base name - position: obj.position, - rotation: obj.rotation, + position: obj.position, + rotation: obj.rotation, scale: obj.scale, isMesh: obj.isMesh, isLight: obj.isLight, From 72937b1dbd86aa05780fe0f8d3c92f37cd4beb6e Mon Sep 17 00:00:00 2001 From: "Mr.doob" Date: Mon, 24 Mar 2025 21:54:33 +0900 Subject: [PATCH 06/12] Improved messageBubble. --- editor/js/Agent.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/editor/js/Agent.js b/editor/js/Agent.js index ef4a01820b2a63..3eac74a15ed0d1 100644 --- a/editor/js/Agent.js +++ b/editor/js/Agent.js @@ -70,6 +70,8 @@ class Agent { messageBubble.style.marginBottom = '8px'; messageBubble.style.fontSize = '14px'; messageBubble.style.position = 'relative'; + messageBubble.style.maxHeight = '400px'; // Add max height + messageBubble.style.overflowY = 'auto'; // Add vertical scrollbar when needed // Add message container first const messageContainer = document.createElement( 'div' ); From 741596b94d2613d4f3a709c714c8395b40362277 Mon Sep 17 00:00:00 2001 From: "Mr.doob" Date: Mon, 24 Mar 2025 21:57:51 +0900 Subject: [PATCH 07/12] Clean up. --- editor/js/Agent.js | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/editor/js/Agent.js b/editor/js/Agent.js index 3eac74a15ed0d1..077ab90fb2e691 100644 --- a/editor/js/Agent.js +++ b/editor/js/Agent.js @@ -179,13 +179,13 @@ class Agent { async init() { - // Initialize Google AI - const ai = new GoogleGenAI( { apiKey: 'AIzaSyCFpCy19tslP3VeSLAPUM-zQjz6Ka9g5no' } ); + // Initialize Google AI + // const session = new GoogleGenAI( { apiKey: 'GENAI_API_KEY' } ); - // Get scene information - const sceneInfo = this.getSceneInfo(); + // Get scene information + const sceneInfo = this.getSceneInfo(); - // Prepare prompt + // Prepare prompt const systemPrompt = `You are a Three.js scene manipulation assistant. Current scene info: ${JSON.stringify( sceneInfo, null, 2 )} From 4968a78659e9b5bd6a88576eb417f0e181db1d95 Mon Sep 17 00:00:00 2001 From: mrdoob Date: Mon, 24 Mar 2025 22:18:34 +0900 Subject: [PATCH 08/12] Potential fix for code scanning alert no. 3445: Unused variable, import, function or class Co-authored-by: Copilot Autofix powered by AI <62310815+github-advanced-security[bot]@users.noreply.github.com> --- editor/js/Agent.js | 1 - 1 file changed, 1 deletion(-) diff --git a/editor/js/Agent.js b/editor/js/Agent.js index 077ab90fb2e691..ef6e14cf9352e6 100644 --- a/editor/js/Agent.js +++ b/editor/js/Agent.js @@ -1,4 +1,3 @@ -import { GoogleGenAI } from '@google/genai'; import * as Commands from './commands/Commands.js'; import { Vector3, BoxGeometry, SphereGeometry, MeshStandardMaterial, Mesh, DirectionalLight, PointLight, AmbientLight, Color, CylinderGeometry } from 'three'; From 22d6b9232201d7c576d003fd840b640821b1ce26 Mon Sep 17 00:00:00 2001 From: "Mr.doob" Date: Mon, 24 Mar 2025 22:20:11 +0900 Subject: [PATCH 09/12] Clean up. --- editor/js/Agent.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/editor/js/Agent.js b/editor/js/Agent.js index ef6e14cf9352e6..8bae5c20b6178a 100644 --- a/editor/js/Agent.js +++ b/editor/js/Agent.js @@ -1,3 +1,4 @@ +import { GoogleGenAI } from '@google/genai'; import * as Commands from './commands/Commands.js'; import { Vector3, BoxGeometry, SphereGeometry, MeshStandardMaterial, Mesh, DirectionalLight, PointLight, AmbientLight, Color, CylinderGeometry } from 'three'; @@ -179,7 +180,7 @@ class Agent { async init() { // Initialize Google AI - // const session = new GoogleGenAI( { apiKey: 'GENAI_API_KEY' } ); + const session = new GoogleGenAI( { apiKey: 'GENAI_API_KEY' } ); // Get scene information const sceneInfo = this.getSceneInfo(); From 9b72ac3352e2d7a212a10b9788542d71c47919ae Mon Sep 17 00:00:00 2001 From: "Mr.doob" Date: Wed, 26 Mar 2025 14:43:06 +0900 Subject: [PATCH 10/12] Clean up. --- editor/js/Agent.js | 50 ++++++++++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 19 deletions(-) diff --git a/editor/js/Agent.js b/editor/js/Agent.js index 8bae5c20b6178a..075ff33ca7cb6c 100644 --- a/editor/js/Agent.js +++ b/editor/js/Agent.js @@ -70,8 +70,8 @@ class Agent { messageBubble.style.marginBottom = '8px'; messageBubble.style.fontSize = '14px'; messageBubble.style.position = 'relative'; - messageBubble.style.maxHeight = '400px'; // Add max height - messageBubble.style.overflowY = 'auto'; // Add vertical scrollbar when needed + messageBubble.style.maxHeight = '400px'; // Add max height + messageBubble.style.overflowY = 'auto'; // Add vertical scrollbar when needed // Add message container first const messageContainer = document.createElement( 'div' ); @@ -91,8 +91,11 @@ class Agent { closeButton.style.lineHeight = '14px'; closeButton.style.zIndex = '1'; closeButton.onclick = () => { + messageBubble.style.display = 'none'; + }; + messageBubble.appendChild( closeButton ); // Create input area @@ -150,37 +153,46 @@ class Agent { // Store references this.messageBubble = messageBubble; + } showError( message ) { + this.showMessage( message, 'error' ); + } showMessage( message, type = 'normal' ) { + // Get the message container const messageContainer = this.messageBubble.querySelector( '.message-text' ); const closeButton = this.messageBubble.querySelector( 'div:last-child' ); // Set styles based on message type if ( type === 'error' ) { + this.messageBubble.style.backgroundColor = '#ffebee'; this.messageBubble.style.color = '#d32f2f'; closeButton.style.color = '#d32f2f'; + } else { + this.messageBubble.style.backgroundColor = '#e8f5e9'; this.messageBubble.style.color = '#2e7d32'; closeButton.style.color = '#2e7d32'; + } // Update message text messageContainer.textContent = message; this.messageBubble.style.display = 'block'; + } async init() { // Initialize Google AI - const session = new GoogleGenAI( { apiKey: 'GENAI_API_KEY' } ); + const ai = new GoogleGenAI( { apiKey: 'GENAI_API_KEY' } ); // Get scene information const sceneInfo = this.getSceneInfo(); @@ -397,27 +409,27 @@ class Agent { Do not include any other text outside the JSON.`; - this.chat = await ai.chats.create({ - model: "gemini-2.0-flash", + this.chat = await ai.chats.create( { + model: 'gemini-2.0-flash', history: [ { - role: "user", - parts: [{ text: systemPrompt }], + role: 'user', + parts: [ { text: systemPrompt } ], }, { - role: "model", - parts: [ - { - text: "I'm ready to help you create and modify your 3D scene.", - }, - ], + role: 'model', + parts: [ + { + text: 'I\'m ready to help you create and modify your 3D scene.', + }, + ], }, ], config: { temperature: 0.2, maxOutputTokens: 2048 }, - }); + } ); console.log( 'CHAT:', this.chat ); @@ -745,7 +757,7 @@ class Agent { // Handle removing textures value = null; - } else if ( typeof value === 'string' && !isNaN( value ) ) { + } else if ( typeof value === 'string' && ! isNaN( value ) ) { // Convert numeric strings to numbers value = parseFloat( value ); @@ -899,11 +911,11 @@ class Agent { } ); const objects = scene.children.map( obj => ( { - type: obj.type, - name: obj.name, + type: obj.type, + name: obj.name, baseName: obj.name.replace( /\d+$/, '' ), // Add base name - position: obj.position, - rotation: obj.rotation, + position: obj.position, + rotation: obj.rotation, scale: obj.scale, isMesh: obj.isMesh, isLight: obj.isLight, From 1cf81928c1a9d82670c8c37dd8a284c9a7c05f37 Mon Sep 17 00:00:00 2001 From: "Mr.doob" Date: Wed, 26 Mar 2025 14:53:09 +0900 Subject: [PATCH 11/12] Test gemini 2.5 pro --- editor/js/Agent.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/editor/js/Agent.js b/editor/js/Agent.js index 075ff33ca7cb6c..ffc510473ecb73 100644 --- a/editor/js/Agent.js +++ b/editor/js/Agent.js @@ -192,7 +192,7 @@ class Agent { async init() { // Initialize Google AI - const ai = new GoogleGenAI( { apiKey: 'GENAI_API_KEY' } ); + const ai = new GoogleGenAI( { apiKey: 'AIzaSyAHEPxUn9Ow4Lk8oXM4jl3u_stzOIxkkGM' } ); // Get scene information const sceneInfo = this.getSceneInfo(); @@ -410,7 +410,7 @@ class Agent { Do not include any other text outside the JSON.`; this.chat = await ai.chats.create( { - model: 'gemini-2.0-flash', + model: 'gemini-2.5-pro-exp-03-25', history: [ { role: 'user', From 10a5a7379afe412b8facc047370ff647cd651cb9 Mon Sep 17 00:00:00 2001 From: "Mr.doob" Date: Wed, 26 Mar 2025 16:44:56 +0900 Subject: [PATCH 12/12] Githack doesn't work if there's a public key? --- editor/js/Agent.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/editor/js/Agent.js b/editor/js/Agent.js index ffc510473ecb73..4ca32571a2399f 100644 --- a/editor/js/Agent.js +++ b/editor/js/Agent.js @@ -192,7 +192,7 @@ class Agent { async init() { // Initialize Google AI - const ai = new GoogleGenAI( { apiKey: 'AIzaSyAHEPxUn9Ow4Lk8oXM4jl3u_stzOIxkkGM' } ); + const ai = new GoogleGenAI( { apiKey: 'GENAI_API_KEY' } ); // Get scene information const sceneInfo = this.getSceneInfo();