diff --git a/main.js b/main.js index 590348c..cabf295 100644 --- a/main.js +++ b/main.js @@ -63,7 +63,7 @@ if (process.env.LOG_ALL) { settings.log_all_prompts = process.env.LOG_ALL; } -Mindcraft.init(false, settings.mindserver_port); +Mindcraft.init(false, settings.mindserver_port, settings.auto_open_browser); for (let profile of settings.profiles) { const profile_json = JSON.parse(readFileSync(profile, 'utf8')); diff --git a/package.json b/package.json index 106a75c..bf0a7c4 100644 --- a/package.json +++ b/package.json @@ -19,6 +19,7 @@ "mineflayer-pathfinder": "^2.4.5", "mineflayer-pvp": "^1.3.2", "node-canvas-webgl": "PrismarineJS/node-canvas-webgl", + "open": "^10.2.0", "openai": "^4.4.0", "patch-package": "^8.0.0", "prismarine-item": "^1.15.0", diff --git a/settings.js b/settings.js index f1454c9..f5487e3 100644 --- a/settings.js +++ b/settings.js @@ -6,6 +6,7 @@ const settings = { // the mindserver manages all agents and hosts the UI "mindserver_port": 8080, + "auto_open_browser": true, "base_profile": "assistant", // survival, assistant, creative, or god_mode "profiles": [ diff --git a/src/agent/library/full_state.js b/src/agent/library/full_state.js new file mode 100644 index 0000000..c5db85f --- /dev/null +++ b/src/agent/library/full_state.js @@ -0,0 +1,89 @@ +import { + getPosition, + getBiomeName, + getNearbyPlayerNames, + getInventoryCounts, + getNearbyEntityTypes, + getNearbyBlockTypes, + getBlockAtPosition, + getFirstBlockAboveHead +} from "./world.js"; +import convoManager from '../conversation.js'; + +export function getFullState(agent) { + const bot = agent.bot; + + const pos = getPosition(bot); + const position = { + x: Number(pos.x.toFixed(2)), + y: Number(pos.y.toFixed(2)), + z: Number(pos.z.toFixed(2)) + }; + + let weather = 'Clear'; + if (bot.thunderState > 0) weather = 'Thunderstorm'; + else if (bot.rainState > 0) weather = 'Rain'; + + let timeLabel = 'Night'; + if (bot.time.timeOfDay < 6000) timeLabel = 'Morning'; + else if (bot.time.timeOfDay < 12000) timeLabel = 'Afternoon'; + + const below = getBlockAtPosition(bot, 0, -1, 0).name; + const legs = getBlockAtPosition(bot, 0, 0, 0).name; + const head = getBlockAtPosition(bot, 0, 1, 0).name; + + let players = getNearbyPlayerNames(bot); + let bots = convoManager.getInGameAgents().filter(b => b !== agent.name); + players = players.filter(p => !bots.includes(p)); + + const helmet = bot.inventory.slots[5]; + const chestplate = bot.inventory.slots[6]; + const leggings = bot.inventory.slots[7]; + const boots = bot.inventory.slots[8]; + + const state = { + name: agent.name, + gameplay: { + position, + dimension: bot.game.dimension, + gamemode: bot.game.gameMode, + health: Math.round(bot.health), + hunger: Math.round(bot.food), + biome: getBiomeName(bot), + weather, + timeOfDay: bot.time.timeOfDay, + timeLabel + }, + action: { + current: agent.isIdle() ? 'Idle' : agent.actions.currentActionLabel, + isIdle: agent.isIdle() + }, + surroundings: { + below, + legs, + head, + firstBlockAboveHead: getFirstBlockAboveHead(bot, null, 32) + }, + inventory: { + counts: getInventoryCounts(bot), + equipment: { + helmet: helmet ? helmet.name : null, + chestplate: chestplate ? chestplate.name : null, + leggings: leggings ? leggings.name : null, + boots: boots ? boots.name : null, + mainHand: bot.heldItem ? bot.heldItem.name : null + } + }, + nearby: { + humanPlayers: players, + botPlayers: bots, + entityTypes: getNearbyEntityTypes(bot).filter(t => t !== 'player' && t !== 'item'), + blockTypes: getNearbyBlockTypes(bot) + }, + modes: { + summary: bot.modes.getMiniDocs() + } + }; + + return state; +} \ No newline at end of file diff --git a/src/agent/mindserver_proxy.js b/src/agent/mindserver_proxy.js index 1426098..2db78e3 100644 --- a/src/agent/mindserver_proxy.js +++ b/src/agent/mindserver_proxy.js @@ -1,6 +1,7 @@ import { io } from 'socket.io-client'; import convoManager from './conversation.js'; import { setSettings } from './settings.js'; +import { getFullState } from './library/full_state.js'; // agent's individual connection to the mindserver // always connect to localhost @@ -65,6 +66,16 @@ class MindServerProxy { } }); + this.socket.on('get-full-state', (callback) => { + try { + const state = getFullState(this.agent); + callback(state); + } catch (error) { + console.error('Error getting full state:', error); + callback(null); + } + }); + // Request settings and wait for response await new Promise((resolve, reject) => { const timeout = setTimeout(() => { diff --git a/src/mindcraft/mindcraft.js b/src/mindcraft/mindcraft.js index 57c4dfe..640576f 100644 --- a/src/mindcraft/mindcraft.js +++ b/src/mindcraft/mindcraft.js @@ -1,6 +1,7 @@ -import { createMindServer, registerAgent } from './mindserver.js'; +import { createMindServer, registerAgent, numStateListeners } from './mindserver.js'; import { AgentProcess } from '../process/agent_process.js'; import { getServer } from './mcserver.js'; +import open from 'open'; let mindserver; let connected = false; @@ -8,7 +9,7 @@ let agent_processes = {}; let agent_count = 0; let port = 8080; -export async function init(host_public=false, port=8080) { +export async function init(host_public=false, port=8080, auto_open_browser=true) { if (connected) { console.error('Already initiliazed!'); return; @@ -16,6 +17,14 @@ export async function init(host_public=false, port=8080) { mindserver = createMindServer(host_public, port); port = port; connected = true; + if (auto_open_browser) { + setTimeout(() => { + // check if browser listener is already open + if (numStateListeners() === 0) { + open('http://localhost:'+port); + } + }, 2000); + } } export async function createAgent(settings) { diff --git a/src/mindcraft/mindserver.js b/src/mindcraft/mindserver.js index acd0775..7f1748a 100644 --- a/src/mindcraft/mindserver.js +++ b/src/mindcraft/mindserver.js @@ -15,6 +15,7 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url)); let io; let server; const agent_connections = {}; +const agent_listeners = []; const settings_spec = JSON.parse(readFileSync(path.join(__dirname, 'public/settings_spec.json'), 'utf8')); @@ -23,8 +24,8 @@ class AgentConnection { this.socket = null; this.settings = settings; this.in_game = false; + this.full_state = null; } - } export function registerAgent(settings) { @@ -114,6 +115,9 @@ export function createMindServer(host_public = false, port = 8080) { agent_connections[curAgentName].in_game = false; agentsUpdate(); } + if (agent_listeners.includes(socket)) { + removeListener(socket); + } }); socket.on('chat-message', (agentName, json) => { @@ -174,6 +178,10 @@ export function createMindServer(host_public = false, port = 8080) { socket.on('bot-output', (agentName, message) => { io.emit('bot-output', agentName, message); }); + + socket.on('listen-to-agents', () => { + addListener(socket); + }); }); let host = host_public ? '0.0.0.0' : 'localhost'; @@ -195,6 +203,42 @@ function agentsUpdate(socket) { socket.emit('agents-update', agents); } + +let listenerInterval = null; +function addListener(listener_socket) { + agent_listeners.push(listener_socket); + if (agent_listeners.length === 1) { + listenerInterval = setInterval(async () => { + const states = {}; + for (let agentName in agent_connections) { + let agent = agent_connections[agentName]; + if (agent.in_game) { + try { + const state = await new Promise((resolve) => { + agent.socket.emit('get-full-state', (s) => resolve(s)); + }); + states[agentName] = state; + } catch (e) { + states[agentName] = { error: String(e) }; + } + } + } + for (let listener of agent_listeners) { + listener.emit('state-update', states); + } + }, 1000); + } +} + +function removeListener(listener_socket) { + agent_listeners.splice(agent_listeners.indexOf(listener_socket), 1); + if (agent_listeners.length === 0) { + clearInterval(listenerInterval); + listenerInterval = null; + } +} + // Optional: export these if you need access to them from other files export const getIO = () => io; export const getServer = () => server; +export const numStateListeners = () => agent_listeners.length; \ No newline at end of file diff --git a/src/mindcraft/public/index.html b/src/mindcraft/public/index.html index d9690a2..3cf7ba4 100644 --- a/src/mindcraft/public/index.html +++ b/src/mindcraft/public/index.html @@ -118,22 +118,115 @@ display: flex; justify-content: flex-start; } + .status-badge { + font-size: 0.75em; + margin-left: 8px; + padding: 2px 6px; + border-radius: 4px; + background: #3a3a3a; + color: #cccccc; + text-transform: lowercase; + } + .status-badge.online { color: #4CAF50; } + .status-badge.offline { color: #f44336; } + .title-row { + display: flex; + align-items: center; + gap: 10px; + } + .title-left { + display: flex; + align-items: center; + gap: 8px; + } + .title-spacer { flex: 1; } + /* Modal styles */ + .modal-backdrop { + position: fixed; + top: 0; left: 0; right: 0; bottom: 0; + background: rgba(0,0,0,0.5); + display: none; + align-items: center; + justify-content: center; + z-index: 1000; + } + .modal { + background: #2d2d2d; + border-radius: 8px; + width: 80vw; + height: 80vh; + display: flex; + flex-direction: column; + overflow: hidden; + box-shadow: 0 4px 12px rgba(0,0,0,0.4); + } + .modal-header { + display: flex; + align-items: center; + justify-content: space-between; + padding: 12px 16px; + border-bottom: 1px solid #3a3a3a; + } + .modal-close-btn { + background: #f44336; + color: #fff; + border: none; + border-radius: 4px; + padding: 4px 8px; + cursor: pointer; + } + .modal-body { + flex: 1 1 auto; + overflow: auto; + padding: 12px 16px; + } + .modal-footer { + display: flex; + align-items: center; + justify-content: space-between; + padding: 12px 16px; + border-top: 1px solid #3a3a3a; + } + .footer-left { color: #cccccc; font-style: italic; } -

Mindcraft

+
+
+

Mindcraft

+ mindserver offline +
+
+
-
-

Create Agent

-
-
Profile: Not uploaded
-
- - - +
+ +
+ + +