Merge pull request #16 from kolbytn/coder

Coder
approved offline by @kolbytn

package.json

@@ -3,6 +3,8 @@
     "dependencies": {
         "minecraft-data": "^3.46.2",
         "mineflayer": "^4.14.0",
+        "mineflayer-armor-manager": "^2.0.1",
+        "mineflayer-auto-eat": "^3.3.6",
         "mineflayer-collectblock": "^1.4.1",
         "mineflayer-pathfinder": "^2.4.4",
         "mineflayer-pvp": "^1.3.2",

src/agent/agent.js

@@ -1,26 +1,31 @@
 import { initBot } from '../utils/mcdata.js';
 import { sendRequest } from '../utils/gpt.js';
 import { History } from './history.js';
+import { Examples } from './examples.js';
 import { Coder } from './coder.js';
-import { getQuery, containsQuery } from './queries.js';
+import { containsCommand, commandExists, executeCommand } from './commands.js';
-import { containsCodeBlock } from './skill-library.js';
 import { Events } from './events.js';
 
 
 export class Agent {
-    constructor(name, profile=null, init_message=null) {
+    async start(name, profile=null, init_message=null) {
         this.name = name;
-        this.bot = initBot(name);
+        this.examples = new Examples();
         this.history = new History(this);
         this.coder = new Coder(this);
 
         this.history.load(profile);
+        await this.examples.load('./src/examples.json');
+        await this.coder.load();
+
+        this.bot = initBot(name);
 
         this.events = new Events(this, this.history.events)
 
         this.bot.on('login', async () => {
-            this.bot.chat('Hello world! I am ' + this.name);
+                
             console.log(`${this.name} logged in.`);
+            this.coder.clear();
             
             const ignore_messages = [
                 "Set own game mode to",
@@ -40,11 +45,17 @@ export class Agent {
                 this.handleMessage(username, message);
             });
 
-            await this.history.loadExamples();
+            // set the bot to automatically eat food when hungry
+            this.bot.autoEat.options = {
+                priority: 'foodPoints',
+                startAt: 14,
+                bannedFood: []
+            };
 
             if (init_message) {
                 this.handleMessage('system', init_message);
             } else {
+                this.bot.chat('Hello world! I am ' + this.name);
                 this.bot.emit('finished_executing');
             }
         });
@@ -54,39 +65,47 @@ export class Agent {
         if (!!source && !!message)
             await this.history.add(source, message);
 
-        for (let i=0; i<5; i++) {
+        const user_command_name = containsCommand(message);
-            let res = await sendRequest(this.history.getHistory(), this.history.getSystemMessage());
+        if (user_command_name) {
-            this.history.add(this.name, res);
+            this.bot.chat(`*${source} used ${user_command_name.substring(1)}*`);
-            let query_cmd = containsQuery(res);
+            let execute_res = await executeCommand(this, message);
-            if (query_cmd) { // contains query
+            if (user_command_name === '!newAction') {
-                let message = res.substring(0, res.indexOf(query_cmd)).trim();
+                // all user initiated commands are ignored by the bot except for this one
-                if (message) 
+                // add the preceding message to the history to give context for newAction
-                    this.bot.chat(message);
+                let truncated_msg = message.substring(0, message.indexOf(user_command_name)).trim();
-                let query = getQuery(query_cmd);
+                this.history.add(source, truncated_msg);
-                let query_res = query.perform(this);
-                console.log('Agent used query:', query_cmd, 'and got:', query_res)
-                this.history.add('system', query_res);
             }
-            else if (containsCodeBlock(res)) { // contains code block
+            if (execute_res)
-                console.log('Agent is executing code:', res)
+                this.bot.chat(execute_res);
+            return;
+        }
 
-                let message = res.substring(0, res.indexOf('```')).trim();
+        for (let i=0; i<5; i++) {
-                if (message) 
+            let history = await this.history.getHistory(this.examples);
-                    this.bot.chat(message);
+            let res = await sendRequest(history, this.history.getSystemMessage());
-                let code = res.substring(res.indexOf('```')+3, res.lastIndexOf('```'));
+            this.history.add(this.name, res);
 
-                if (code) {
+            let command_name = containsCommand(res);
-                    this.coder.queueCode(code);
+
-                    let code_return = await this.coder.execute();
+            if (command_name) { // contains query or command
-                    let message = code_return.message;
+                console.log('Command message:', res);
-                    if (code_return.interrupted && !code_return.timedout)
+                if (!commandExists(command_name)) {
-                        break;
+                    this.history.add('system', `Command ${command_name} does not exist. Use !newAction to perform custom actions.`);
-                    if (!code_return.success) {
+                    console.log('Agent hallucinated command:', command_name)
-                        message += "\nWrite code to fix the problem and try again.";
+                    continue;
-                    }
-                    console.log('code return:', message);
-                    this.history.add('system', message);
                 }
+
+                let pre_message = res.substring(0, res.indexOf(command_name)).trim();
+
+                this.bot.chat(`${pre_message}  *used ${command_name.substring(1)}*`);
+                let execute_res = await executeCommand(this, res);
+
+                console.log('Agent executed:', command_name, 'and got:', execute_res);
+
+                if (execute_res)
+                    this.history.add('system', execute_res);
+                else
+                    break;
             }
             else { // conversation response
                 this.bot.chat(res);

src/agent/coder.js

@@ -1,17 +1,23 @@
-import { writeFile, readFile, unlink, mkdirSync } from 'fs';
+import { writeFile, readFile, mkdirSync } from 'fs';
+import { sendRequest } from '../utils/gpt.js';
+import { getSkillDocs } from './skill-library.js';
+import { Examples } from './examples.js';
+
 
 export class Coder {
     constructor(agent) {
         this.agent = agent;
-        this.queued_code = '';
         this.current_code = '';
         this.file_counter = 0;
         this.fp = '/bots/'+agent.name+'/action-code/';
-        this.agent.bot.interrupt_code = false;
         this.executing = false;
-        this.agent.bot.output = '';
         this.code_template = '';
         this.timedout = false;
+    }
+
+    async load() {
+        this.examples = new Examples();
+        await this.examples.load('./src/examples_coder.json');
 
         readFile('./bots/template.js', 'utf8', (err, data) => {
             if (err) throw err;
@@ -21,8 +27,40 @@ export class Coder {
         mkdirSync('.' + this.fp, { recursive: true });
     }
 
-    queueCode(code) {
+    // write custom code to file and import it
-        this.queued_code = this.santitizeCode(code);
+    async stageCode(code) {
+        code = this.santitizeCode(code);
+        let src = '';
+        code = code.replaceAll('console.log(', 'log(bot,');
+        code = code.replaceAll('log("', 'log(bot,"');
+
+        // this may cause problems in callback functions
+        code = code.replaceAll(';\n', '; if(bot.interrupt_code) {log(bot, "Code interrupted.");return;}\n');
+        for (let line of code.split('\n')) {
+            src += `    ${line}\n`;
+        }
+        src = this.code_template.replace('/* CODE HERE */', src);
+
+        console.log("writing to file...", src)
+
+        let filename = this.file_counter + '.js';
+        // if (this.file_counter > 0) {
+        //     let prev_filename = this.fp + (this.file_counter-1) + '.js';
+        //     unlink(prev_filename, (err) => {
+        //         console.log("deleted file " + prev_filename);
+        //         if (err) console.error(err);
+        //     });
+        // } commented for now, useful to keep files for debugging
+        this.file_counter++;
+
+        let write_result = await this.writeFilePromise('.' + this.fp + filename, src)
+        
+        if (write_result) {
+            console.error('Error writing code execution file: ' + result);
+            return null;
+        }
+        this.current_code = code;
+        return await import('../..' + this.fp + filename);
     }
 
     santitizeCode(code) {
@@ -50,51 +88,81 @@ export class Coder {
     }
 
 
-    // returns {success: bool, message: string, interrupted: bool, timedout: false}
+    async generateCode(agent_history) {
-    async execute() {
+        let system_message = "You are a minecraft bot that plays minecraft by writing javascript codeblocks. Given the conversation between you and the user, use the provided skills and world queries to write your code in a codeblock. Example response: ``` // your code here ``` You will then be given a response to your code. If you are satisfied with the response, respond without a codeblock in a conversational way. If something went wrong, write another codeblock and try to fix the problem.";
-        if (!this.queued_code) return {success: false, message: "No code to execute.", interrupted: false, timedout: false};
+        system_message += getSkillDocs();
-        if (!this.code_template) return {success: false, message: "Code template not loaded.", interrupted: false, timedout: false};
-        let src = '';
 
-        let code = this.queued_code;
+        system_message += "\n\nExamples:\nUser zZZn98: come here \nAssistant: I am going to navigate to zZZn98. ```\nawait skills.goToPlayer(bot, 'zZZn98');```\nSystem: Code execution finished successfully.\nAssistant: Done.";
-        code = code.replaceAll('console.log(', 'log(bot,');
-        code = code.replaceAll('log("', 'log(bot,"');
 
-        // this may cause problems in callback functions
+        let messages = await agent_history.getHistory(this.examples);
-        code = code.replaceAll(';\n', '; if(bot.interrupt_code) {log(bot, "Code interrupted.");return;}\n');
+
-        for (let line of code.split('\n')) {
+        let code_return = null;
-            src += `    ${line}\n`;
+        let failures = 0;
+        for (let i=0; i<5; i++) {
+            console.log(messages)
+            let res = await sendRequest(messages, system_message);
+            console.log('Code generation response:', res)
+            let contains_code = res.indexOf('```') !== -1;
+            if (!contains_code) {
+                if (code_return) {
+                    agent_history.add('system', code_return.message);
+                    agent_history.add(this.agent.name, res);
+                    this.agent.bot.chat(res);
+                    return;
+                }
+                if (failures >= 1) {
+                    agent_history.add('system', 'Action failed, agent would not write code.');
+                    return;
+                }
+                messages.push({
+                    role: 'system', 
+                    content: 'Error: no code provided. Write code in codeblock in your response. ``` // example ```'}
+                );
+                failures++;
+                continue;
+            }
+            let code = res.substring(res.indexOf('```')+3, res.lastIndexOf('```'));
+
+            const execution_file = await this.stageCode(code);
+            if (!execution_file) {
+                agent_history.add('system', 'Failed to stage code, something is wrong.');
+                return;
+            }
+            code_return = await this.execute(async ()=>{
+                return await execution_file.main(this.agent.bot);
+            });
+
+            if (code_return.interrupted && !code_return.timedout)
+                return;
+            console.log(code_return.message);
+
+            messages.push({
+                role: 'assistant',
+                content: res
+            });
+            messages.push({
+                role: 'system',
+                content: code_return.message
+            });
         }
-        src = this.code_template.replace('/* CODE HERE */', src);
-
-        console.log("writing to file...", src)
-
-        let filename = this.file_counter + '.js';
-        // if (this.file_counter > 0) {
-        //     let prev_filename = this.fp + (this.file_counter-1) + '.js';
-        //     unlink(prev_filename, (err) => {
-        //         console.log("deleted file " + prev_filename);
-        //         if (err) console.error(err);
-        //     });
-        // } commented for now, useful to keep files for debugging
-        this.file_counter++;
-
-        let write_result = await this.writeFilePromise('.' + this.fp + filename, src)
         
-        if (write_result) {
+        return 
-            console.error('Error writing code execution file: ' + result);
+    }
-            return {success: false, message: result, interrupted: false, timedout: false};
+
-        }
+    // returns {success: bool, message: string, interrupted: bool, timedout: false}
+    async execute(func, timeout=10) {
+        if (!this.code_template) return {success: false, message: "Code template not loaded.", interrupted: false, timedout: false};
+
         let TIMEOUT;
         try {
             console.log('executing code...\n');
-            let execution_file = await import('../..' + this.fp + filename);
             await this.stop();
-            this.current_code = this.queued_code;
+            this.clear();
 
             this.executing = true;
-            TIMEOUT = this._startTimeout(10);
+            if (timeout > 0)
-            await execution_file.main(this.agent.bot); // open fire
+                TIMEOUT = this._startTimeout(timeout);
+            await func(); // open fire
             this.executing = false;
             clearTimeout(TIMEOUT);
 
@@ -109,10 +177,11 @@ export class Coder {
             clearTimeout(TIMEOUT);
 
             console.error("Code execution triggered catch: " + err);
+            await this.stop();
             let message = this.formatOutput(this.agent.bot);
             message += '!!Code threw exception!!  Error: ' + err;
             let interrupted = this.agent.bot.interrupt_code;
-            await this.stop();
+            this.clear();
             this.agent.bot.emit("code_terminated");
             return {success: false, message, interrupted, timedout: false};
         }
@@ -142,7 +211,6 @@ export class Coder {
             console.log('waiting for code to finish executing... interrupt:', this.agent.bot.interrupt_code);
             await new Promise(resolve => setTimeout(resolve, 1000));
         }
-        this.clear();
     }
 
     clear() {

src/agent/commands.js

@@ -0,0 +1,101 @@
+
+import { actionsList } from './commands/actions.js';
+import { queryList } from './commands/queries.js';
+
+const commandList = queryList.concat(actionsList);
+const commandMap = {};
+for (let command of commandList) {
+    commandMap[command.name] = command;
+}
+
+export function getCommand(name) {
+    return commandMap[name];
+}
+
+const commandRegex = /!(\w+)(?:\(((?:[^)(]+|'[^']*'|"[^"]*")*)\))?/
+const argRegex = /(?:"[^"]*"|'[^']*'|[^,])+/g;
+
+export function containsCommand(message) {
+    const commandMatch = message.match(commandRegex);
+    if (commandMatch)
+        return "!" + commandMatch[1];
+    return null;
+}
+
+export function commandExists(commandName) {
+    if (!commandName.startsWith("!"))
+        commandName = "!" + commandName;
+    return commandMap[commandName] !== undefined;
+}
+
+// todo: handle arrays?
+function parseCommandMessage(message) {
+    const commandMatch = message.match(commandRegex);
+    if (commandMatch) {
+        const commandName = "!"+commandMatch[1];
+        if (!commandMatch[2])
+            return { commandName, args: [] };
+        let args = commandMatch[2].match(argRegex);
+        if (args) {
+            for (let i = 0; i < args.length; i++) {
+                args[i] = args[i].trim();
+            }
+
+            for (let i = 0; i < args.length; i++) {
+                let arg = args[i];
+                if ((arg.startsWith('"') && arg.endsWith('"')) || (arg.startsWith("'") && arg.endsWith("'"))) {
+                    args[i] = arg.substring(1, arg.length-1);
+                } else if (!isNaN(arg)) {
+                    args[i] = Number(arg);
+                } else if (arg === 'true' || arg === 'false') {
+                    args[i] = arg === 'true';
+                }
+            }
+        }
+        else
+            args = [];
+
+        return { commandName, args };
+    }
+    return null;
+}
+
+function numParams(command) {
+    if (!command.params)
+        return 0;
+    return Object.keys(command.params).length;
+}
+
+export async function executeCommand(agent, message) {
+    let parsed = parseCommandMessage(message);
+    if (parsed) {
+        const command = getCommand(parsed.commandName);
+        let numArgs = 0;
+        if (parsed.args) {
+            numArgs = parsed.args.length;
+        }
+        console.log('parsed command:', parsed);
+        if (numArgs !== numParams(command))
+            return `Command ${command.name} was given ${numArgs} args, but requires ${numParams(command)} args.`;
+        else
+            return await command.perform(agent, ...parsed.args);
+    }
+    else
+        return `Command is incorrectly formatted`;
+}
+
+export function getCommandDocs() {
+    let docs = `\n*COMMAND DOCS\n You can use the following commands to perform actions and get information about the world. 
+    Use the commands with the syntax: !commandName or !commandName("arg1", 1.2, ...) if the command takes arguments.\n
+    Do not use codeblocks. Only use one command in each response, trailing commands and comments will be ignored. Use these commands frequently in your responses!\n`;
+    for (let command of commandList) {
+        docs += command.name + ': ' + command.description + '\n';
+        if (command.params) {
+            docs += 'Params:\n';
+            for (let param in command.params) {
+                docs += param + ': ' + command.params[param] + '\n';
+            }
+        }
+    }
+    return docs + '*\n';
+}

src/agent/commands/actions.js

@@ -0,0 +1,105 @@
+import * as skills from '../skills.js';
+import * as world from '../world.js';
+
+function wrapExecution(func) {
+    return async function (agent, ...args) {
+        let code_return = await agent.coder.execute(async () => {
+            await func(agent, ...args);
+        }, -1); // no timeout
+        if (code_return.interrupted && !code_return.timedout)
+            return;
+        return code_return.message;
+    }
+}
+
+export const actionsList = [
+    {
+        name: '!newAction',
+        description: 'Perform new and unknown custom behaviors that are not available as a command by writing code.', 
+        perform: async function (agent) {
+            let res = await agent.coder.generateCode(agent.history);
+            if (res)
+                return '\n' + res + '\n';
+        }
+    },
+    {
+        name: '!stop',
+        description: 'Force stop all actions and commands that are currently executing.',
+        perform: async function (agent) {
+            await agent.coder.stop();
+            return 'Agent stopped.';
+        }
+    },
+    {
+        name: '!goToPlayer',
+        description: 'Go to the given player. Ex: !goToPlayer("steve")',
+        params: {'player_name': '(string) The name of the player to go to.'},
+        perform: wrapExecution(async (agent, player_name) => {
+            return await skills.goToPlayer(agent.bot, player_name);
+        })
+    },
+    {
+        name: '!followPlayer',
+        description: 'Endlessly follow the given player. Ex: !followPlayer("stevie")',
+        params: {'player_name': '(string) The name of the player to follow.'},
+        perform: wrapExecution(async (agent, player_name) => {
+            await skills.followPlayer(agent.bot, player_name);
+        })
+    },
+    {
+        name: '!collectBlocks',
+        description: 'Collect the nearest blocks of a given type.',
+        params: {
+            'type': '(string) The block type to collect. Ex: !collectBlocks("stone", 10)',
+            'num': '(number) The number of blocks to collect.'
+        },
+        perform: wrapExecution(async (agent, type, num) => {
+            await skills.collectBlock(agent.bot, type, num);
+        })
+    },
+    {
+        name: '!craftRecipe',
+        description: 'Craft the given recipe a given number of times. Ex: I will craft 8 sticks !craftRecipe("stick", 2)',
+        params: {
+            'recipe_name': '(string) The name of the output item to craft.',
+            'num': '(number) The number of times to craft the recipe. This is NOT the number of output items, as it may craft many more items depending on the recipe.'
+        },
+        perform: wrapExecution(async (agent, recipe_name, num) => {
+            for (let i=0; i<num; i++) {
+                await skills.craftRecipe(agent.bot, recipe_name);
+            }
+        })
+    },
+    {
+        name: '!placeHere',
+        description: 'Place a given block in the current location. Do NOT use to build structures, only use for single blocks/torches. Ex: !placeBlockHere("crafting_table")',
+        params: {'type': '(string) The block type to place.'},
+        perform: wrapExecution(async (agent, type) => {
+            let pos = agent.bot.entity.position;
+            await skills.placeBlock(agent.bot, type, pos.x, pos.y, pos.z);
+        })
+    },
+    {
+        name: '!attack',
+        description: 'Attack and kill the nearest entity of a given type.',
+        params: {'type': '(string) The type of entity to attack.'},
+        perform: wrapExecution(async (agent, type) => {
+            await skills.attackMob(agent.bot, type, true);
+        })
+    },
+    {
+        name: '!defend',
+        description: 'Follow the given player and attack any nearby monsters.',
+        params: {'player_name': '(string) The name of the player to defend.'},
+        perform: wrapExecution(async (agent, player_name) => {
+            await skills.defendPlayer(agent.bot, player_name);
+        })
+    },
+    {
+        name: '!goToBed',
+        description: 'Go to the nearest bed and sleep.',
+        perform: wrapExecution(async (agent) => {
+            await skills.goToBed(agent.bot);
+        })
+    }
+];

src/agent/commands/queries.js

@@ -1,26 +1,35 @@
-import { getNearestBlock, getNearbyMobTypes, getNearbyPlayerNames, getNearbyBlockTypes, getInventoryCounts } from './world.js';
+import { getNearestBlock, getNearbyMobTypes, getNearbyPlayerNames, getNearbyBlockTypes, getInventoryCounts } from '../world.js';
-import { getAllItems } from '../utils/mcdata.js';
+import { getAllItems, getBiomeName } from '../../utils/mcdata.js';
-
 
 const pad = (str) => {
     return '\n' + str + '\n';
 }
 
-const queryList = [
+// queries are commands that just return strings and don't affect anything in the world
+export const queryList = [
     {
         name: "!stats",
-        description: "Get your bot's stats", 
+        description: "Get your bot's location, health, hunger, and time of day.", 
         perform: function (agent) {
             let bot = agent.bot;
             let res = 'STATS';
-            res += `\n- position: x:${bot.entity.position.x}, y:${bot.entity.position.y}, z:${bot.entity.position.z}`;
+            let pos = bot.entity.position;
-            res += `\n- health: ${bot.health} / 20`;
+            // display position to 2 decimal places
+            res += `\n- Position: x: ${pos.x.toFixed(2)}, y: ${pos.y.toFixed(2)}, z: ${pos.z.toFixed(2)}`;
+            res += `\n- Health: ${Math.round(bot.health)} / 20`;
+            res += `\n- Hunger: ${Math.round(bot.food)} / 20`;
+            res += `\n- Biome: ${getBiomeName(bot)}`;
+            // let block = bot.blockAt(pos);
+            // res += `\n- Artficial light: ${block.skyLight}`;
+            // res += `\n- Sky light: ${block.light}`;
+            // light properties are bugged, they are not accurate
+
             if (bot.time.timeOfDay < 6000) {
-                res += '\n- time: Morning';
+                res += '\n- Time: Morning';
             } else if (bot.time.timeOfDay < 12000) {
-                res += '\n- time: Afternoon';
+                res += '\n- Time: Afternoon';
             } else {
-                res += '\n- time: Night';
+                res += '\n- Time: Night';
             }
             return pad(res);
         }
@@ -96,36 +105,10 @@ const queryList = [
         }
     },
     {
-        name: "!action",
+        name: "!currentAction",
         description: "Get the currently executing code.",
         perform: function (agent) {
             return pad("Current code:\n`" + agent.coder.current_code +"`");
         }
     }
-];
+];
-
-const queryMap = {};
-for (let query of queryList) {
-    queryMap[query.name] = query;
-}
-
-export function getQuery(name) {
-    return queryMap[name];
-}
-
-export function containsQuery(message) {
-    for (let query of queryList) {
-        if (message.includes(query.name)) {
-            return query.name;
-        }
-    }
-    return null;
-}
-
-export function getQueryDocs() {
-    let docs = `\n*QUERY DOCS\n You can use the following commands to query for information about the world. Use the query name in your response and the next input will have the requested information.\n`;
-    for (let query of queryList) {
-        docs += query.name + ': ' + query.description + '\n';
-    }
-    return docs + '*\n';
-}

src/agent/examples.js

@@ -0,0 +1,64 @@
+import { readFileSync } from 'fs';
+import {  embed, cosineSimilarity } from '../utils/gpt.js';
+import { stringifyTurns } from '../utils/text.js';
+
+
+export class Examples {
+    constructor(select_num=3) {
+        this.examples = [];
+        this.select_num = select_num;
+    }
+
+    async load(path) {
+        let examples = [];
+        try {
+            const data = readFileSync(path, 'utf8');
+            examples = JSON.parse(data);
+        } catch (err) {
+            console.error('Examples failed to load!', err);
+        }
+
+        this.examples = [];
+        for (let example of examples) {
+            let messages = '';
+            for (let turn of example) {
+                if (turn.role != 'assistant')
+                    messages += turn.content.substring(turn.content.indexOf(':')+1).trim() + '\n';
+            }
+            messages = messages.trim();
+            const embedding = await embed(messages);
+            this.examples.push({'embedding': embedding, 'turns': example});
+        }
+    }
+
+    async getRelevant(turns) {
+        let messages = '';
+        for (let turn of turns) {
+            if (turn.role != 'assistant')
+                messages += turn.content.substring(turn.content.indexOf(':')+1).trim() + '\n';
+        }
+        messages = messages.trim();
+        const embedding = await embed(messages);
+        this.examples.sort((a, b) => {
+            return cosineSimilarity(b.embedding, embedding) - cosineSimilarity(a.embedding, embedding);
+        });
+        let selected = this.examples.slice(0, this.select_num);
+        return JSON.parse(JSON.stringify(selected)); // deep copy
+    }
+
+    async createExampleMessage(turns) {
+        let selected_examples = await this.getRelevant(turns);
+
+        console.log('selected examples:');
+        for (let example of selected_examples) {
+            console.log(example.turns[0].content)
+        }
+
+        let msg = 'Here are some examples of how to respond:\n';
+        for (let i=0; i<selected_examples.length; i++) {
+            let example = selected_examples[i];
+            msg += `Example ${i+1}:\n${stringifyTurns(example.turns)}\n\n`;
+        }
+        return [{'role': 'system', 'content': msg}];
+    }
+}

src/agent/history.js

@@ -1,7 +1,7 @@
 import { writeFileSync, readFileSync, mkdirSync } from 'fs';
-import { getQueryDocs } from './queries.js';
+import { getCommandDocs } from './commands.js';
-import { getSkillDocs } from './skill-library.js';
+import { sendRequest } from '../utils/gpt.js';
-import { sendRequest, embed, cosineSimilarity } from '../utils/gpt.js';
+import { stringifyTurns } from '../utils/text.js';
 
 
 export class History {
@@ -19,27 +19,22 @@ export class History {
 
         // Variables for controlling the agent's memory and knowledge
         this.max_messages = 20;
-        this.fewshot = 5;
-        this.examples = [];
-        this.selected_examples = [];
     }
 
-    getHistory(include_examples=true) {
+    async getHistory(examples=null) { // expects an Examples object
-        let history = [];
+        let turns = JSON.parse(JSON.stringify(this.turns));
-        if (include_examples && this.selected_examples.length > 0) {
+        if (examples) {
-            for (let example of this.selected_examples) {
+            let examples_msg = await examples.createExampleMessage(turns);
-                history = history.concat(example.turns);
+            turns = examples_msg.concat(turns);
-            }
         }
-        history = history.concat(this.turns);
+        
-
+        return turns;
-        return history;
     }
 
     getSystemMessage() {
-        let system_message = `You are a playful Minecraft bot named '${this.name}' that can communicate with players, see, move, mine, build, and interact with the world by writing and executing code. Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, omit needless words, and do not give instructions unless asked.`;
+        let system_message = `You are a playful Minecraft bot named '${this.name}' that can communicate with players, see, move, mine, build, and interact with the world by using commands. Act human-like as if you were a typical Minecraft player, rather than an AI. Be very brief in your responses, use commands often, and do not give instructions unless asked.
-        system_message += getQueryDocs();
+        Don't pretend to act, use commands immediately when requested. Do NOT do this: "Sure, I'll follow you! *follows you*", instead do this: "Sure I'll follow you! !followPlayer('steve')". Have fun :) \n`;
-        system_message += getSkillDocs();
+        system_message += getCommandDocs();
         if (this.bio != '')
             system_message += '\n\nBio:\n' + this.bio;
         if (this.memory != '')
@@ -48,21 +43,6 @@ export class History {
         return system_message;
     }
 
-    stringifyTurns(turns) {
-        let res = '';
-        for (let turn of turns) {
-            if (turn.role === 'assistant') {
-                res += `\nYour output:\n${turn.content}`;
-            } else if (turn.role === 'system') {
-                res += `\nSystem output: ${turn.content}`;
-            } else {
-                res += `\nUser input: ${turn.content}`;
-            
-            }
-        }
-        return res.trim();
-    }
-
     async storeMemories(turns) {
         console.log("To summarize:", turns)
         let memory_prompt = 'Update your "Memory" by summarizing the following conversation. Your "Memory" is for storing information that will help you improve as a Minecraft bot. Include details about your interactions with other players that you may need to remember for later. Also include things that you have learned through player feedback or by executing code. Do not include information found in your Docs or that you got right on the first try. Be extremely brief and clear.';
@@ -75,7 +55,7 @@ export class History {
         memory_prompt += '- I learned that player [name]...\n';
         
         memory_prompt += 'This is the conversation to summarize:\n';
-        memory_prompt += this.stringifyTurns(turns);
+        memory_prompt += stringifyTurns(turns);
 
         memory_prompt += 'Summarize relevant information from your previous memory and this conversation:\n';
 
@@ -83,47 +63,6 @@ export class History {
         this.memory = await sendRequest(memory_turns, this.getSystemMessage());
     }
 
-    async loadExamples() {
-        let examples = [];
-        try {
-            const data = readFileSync('./src/examples.json', 'utf8');
-            examples = JSON.parse(data);
-        } catch (err) {
-            console.log('No history examples found.');
-        }
-
-        this.examples = [];
-        for (let example of examples) {
-            let messages = '';
-            for (let turn of example) {
-                if (turn.role != 'assistant')
-                    messages += turn.content.substring(turn.content.indexOf(':')+1).trim() + '\n';
-            }
-            messages = messages.trim();
-            const embedding = await embed(messages);
-            this.examples.push({'embedding': embedding, 'turns': example});
-        }
-
-        await this.setExamples();
-    }
-
-    async setExamples() {
-        let messages = '';
-        for (let turn of this.turns) {
-            if (turn.role != 'assistant')
-                messages += turn.content.substring(turn.content.indexOf(':')+1).trim() + '\n';
-        }
-        messages = messages.trim();
-        const embedding = await embed(messages);
-        this.examples.sort((a, b) => {
-            return cosineSimilarity(a.embedding, embedding) - cosineSimilarity(b.embedding, embedding);
-        });
-        this.selected_examples = this.examples.slice(-this.fewshot);
-        for (let example of this.selected_examples) {
-            console.log('selected example: ', example.turns[0].content);
-        }
-    }
-
     async add(name, content) {
         let role = 'assistant';
         if (name === 'system') {
@@ -143,9 +82,6 @@ export class History {
                 to_summarize.push(this.turns.shift());
             await this.storeMemories(to_summarize);
         }
-
-        if (role != 'assistant')
-            await this.setExamples();
     }
 
     save() {

src/agent/skill-library.js

@@ -2,7 +2,7 @@ import * as skills from './skills.js';
 import * as world from './world.js';
 
 export function getSkillDocs() {
-    let docstring = "\n*SKILL DOCS\nThese skills are javascript functions that can be called with a js function by writing a code block. Ex: '```// write description comment and code here```' \nYour code block should return a bool indicating if the task was completed successfully. It will return true if you don't write a return statement.\n";
+    let docstring = "\n*SKILL DOCS\nThese skills are javascript functions that can be called when writing actions and skills.\n";
     docstring += docHelper(Object.values(skills), 'skills');
     docstring += docHelper(Object.values(world), 'world');
     return docstring + '*\n';

src/agent/skills.js

@@ -13,7 +13,7 @@ export async function craftRecipe(bot, itemName) {
      * Attempt to craft the given item name from a recipe. May craft many items.
      * @param {MinecraftBot} bot, reference to the minecraft bot.
      * @param {string} itemName, the item name to craft.
-     * @returns {Promise<boolean>} true if the item was crafted, false otherwise.
+     * @returns {Promise<boolean>} true if the recipe was crafted, false otherwise.
      * @example
      * await skills.craftRecipe(bot, "stick");
      **/
@@ -44,13 +44,15 @@ export async function smeltItem(bot, itemName, num=1) {
     /**
      * Puts 1 coal in furnace and smelts the given item name, waits until the furnace runs out of fuel or input items.
      * @param {MinecraftBot} bot, reference to the minecraft bot.
-     * @param {string} itemName, the item name to smelt. Must contain "raw"
+     * @param {string} itemName, the item name to smelt. Ores must contain "raw" like raw_iron.
      * @param {number} num, the number of items to smelt. Defaults to 1.
      * @returns {Promise<boolean>} true if the item was smelted, false otherwise. Fail
      * @example
      * await skills.smeltItem(bot, "raw_iron");
+     * await skills.smeltItem(bot, "beef");
      **/
-    if (!itemName.includes('raw')) {
+    const foods = ['beef', 'chicken', 'cod', 'mutton', 'porkchop', 'rabbit', 'salmon', 'tropical_fish'];
+    if (!itemName.includes('raw') && !foods.includes(itemName)) {
         log(bot, `Cannot smelt ${itemName}, must be a "raw" item, like "raw_iron".`);
         return false;
     } // TODO: allow cobblestone, sand, clay, etc.
@@ -166,6 +168,13 @@ export async function clearNearestFurnace(bot) {
 }
 
 
+function equipHighestAttack(bot) {
+    let weapons = bot.inventory.items().filter(item => item.name.includes('sword') || item.name.includes('axe') || item.name.includes('pickaxe') || item.name.includes('shovel'));
+    let weapon = weapons.sort((a, b) => b.attackDamage - a.attackDamage)[0];
+    if (weapon)
+        bot.equip(weapon, 'hand');
+}
+
 export async function attackMob(bot, mobType, kill=true) {
     /**
      * Attack mob of the given type.
@@ -177,16 +186,11 @@ export async function attackMob(bot, mobType, kill=true) {
      * await skills.attackMob(bot, "zombie", true);
      **/
     const mob = bot.nearestEntity(entity => entity.name && entity.name.toLowerCase() === mobType.toLowerCase());
-    const attackable = ['animal', 'monster', 'mob'];
+    if (mob) {
-    if (mob && attackable.includes(mob.type)) {
         let pos = mob.position;
         console.log(bot.entity.position.distanceTo(pos))
 
-        // equip highest damage weapon
+        equipHighestAttack(bot)
-        let weapons = bot.inventory.items().filter(item => item.name.includes('sword') || item.name.includes('axe') || item.name.includes('pickaxe') || item.name.includes('shovel'));
-        let weapon = weapons.sort((a, b) => b.attackDamage - a.attackDamage)[0];
-        if (weapon)
-            await bot.equip(weapon, 'hand');
 
         if (!kill) {
             if (bot.entity.position.distanceTo(pos) > 5) {
@@ -542,13 +546,10 @@ export async function goToPlayer(bot, username) {
         return false;
     }
     
-    let arrived = await goToPosition(bot, player.position.x, player.position.y, player.position.z);
+    bot.pathfinder.setMovements(new pf.Movements(bot));
-    if (!arrived) {
+    await bot.pathfinder.goto(new pf.goals.GoalFollow(player, 2), true);
-        log(bot, `Failed to reach ${username}.`);
+
-        return false;
+    log(bot, `You have reached ${username}.`);
-    }
-    log(bot, `Player is now at ${player.position}.`);
-    return true;
 }
 
 
@@ -574,4 +575,89 @@ export async function followPlayer(bot, username) {
     }
 
     return true;
-}
+}
+
+export async function defendPlayer(bot, username) {
+    /**
+     * Defend the given player endlessly, attacking any nearby monsters. Will not return until the code is manually stopped.
+     * @param {MinecraftBot} bot, reference to the minecraft bot.
+     * @param {string} username, the username of the player to defend.
+     * @returns {Promise<boolean>} true if the player was found, false otherwise.
+     * @example
+     * await skills.defendPlayer(bot, "bob");
+     **/
+    let player = bot.players[username].entity
+    if (!player)
+        return false;
+
+    const follow_distance = 3;
+    const attack_distance = 12;
+    const return_distance = 16;
+
+    bot.pathfinder.setMovements(new pf.Movements(bot));
+    bot.pathfinder.setGoal(new pf.goals.GoalFollow(player, follow_distance), true);
+    log(bot, `Actively defending player ${username}.`);
+
+    while (!bot.interrupt_code) {
+        if (bot.entity.position.distanceTo(player.position) < return_distance) {
+            const mobs = getNearbyMobs(bot, attack_distance).filter(mob => mob.type === 'mob' || mob.type === 'hostile');
+            const mob = mobs.sort((a, b) => a.position.distanceTo(player.position) - b.position.distanceTo(player.position))[0]; // get closest to player
+            if (mob) {
+                bot.pathfinder.stop();
+                log(bot, `Found ${mob.name}, attacking!`);
+                bot.chat(`Found ${mob.name}, attacking!`);
+                equipHighestAttack(bot);
+                bot.pvp.attack(mob);
+                while (getNearbyMobs(bot, attack_distance).includes(mob)) {
+                    await new Promise(resolve => setTimeout(resolve, 500));
+                    console.log('attacking...')
+                    if (bot.interrupt_code)
+                        return;
+                    if (bot.entity.position.distanceTo(player.position) > return_distance) {
+                        console.log('stopping pvp...');
+                        bot.pvp.stop();
+                        break;
+                    }
+                }
+                console.log('resuming pathfinder...')
+                bot.pathfinder.setMovements(new pf.Movements(bot));
+                bot.pathfinder.setGoal(new pf.goals.GoalFollow(player, 5), true);
+                await new Promise(resolve => setTimeout(resolve, 3000));
+            }
+        }
+        await new Promise(resolve => setTimeout(resolve, 500));
+    }
+
+    return true;
+}
+
+export async function goToBed(bot) {
+    /**
+     * Sleep in the nearest bed.
+     * @param {MinecraftBot} bot, reference to the minecraft bot.
+     * @returns {Promise<boolean>} true if the bed was found, false otherwise.
+     * @example
+     * await skills.goToBed(bot);
+     **/
+    const beds = bot.findBlocks({
+        matching: (block) => {
+            return block.name.includes('bed');
+        },
+        maxDistance: 32,
+        count: 1
+    });
+    if (beds.length === 0) {
+        log(bot, `Could not find a bed to sleep in.`);
+        return false;
+    }
+    let loc = beds[0];
+    await goToPosition(bot, loc.x, loc.y, loc.z);
+    const bed = bot.blockAt(loc);
+    await bot.sleep(bed);
+    log(bot, `You are in bed.`);
+    while (bot.isSleeping) {
+        await new Promise(resolve => setTimeout(resolve, 500));
+    }
+    log(bot, `You have woken up.`);
+    return true;
+}

src/examples.json

@@ -15,56 +15,60 @@
         {"role": "user", "content": "greg: Collect 10 wood"},
         {"role": "assistant", "content": "Let me see what's nearby... !blocks"},
         {"role": "system", "content": "NEARBY_BLOCKS\n- oak_log\n- dirt\n- cobblestone"},
-        {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone. I'll collect oak logs.\n```\n// I am going to collect 10 oak_log.\nawait skills.collectBlock(bot, 'oak_log', 10);\n```"}
+        {"role": "assistant", "content": "I see some oak logs, dirt, and cobblestone. I'll collect oak logs. !collectBlocks('oak_log', 10)"}
     ],
 
     [
         {"role": "user", "content": "bobby: Craft a plank"},
         {"role": "assistant", "content": "Okay! !craftable"},
         {"role": "system", "content": "CRAFTABLE_ITEMS\n- spruce_planks\n- spruce_wood\n- ladder"},
-        {"role": "assistant", "content": "```\n// I am going to craft a spruce_planks.\nawait skills.craftRecipe(bot, 'spruce_planks');\n```"}
+        {"role": "assistant", "content": "!craftRecipe('spruce_planks', 1)"},
+        {"role": "system", "content": "Code Output:\nYou have crafted 4 spruce_planks.\nCode execution finished successfully."},
+        {"role": "assistant", "content": "I've crafted 4 spruce planks!"}
     ],
 
     [
         {"role": "user", "content": "zZZn98: come here"},
-        {"role": "assistant", "content": "```\n// I am going to navigate to zZZn98.\nawait skills.goToPlayer(bot, 'zZZn98');\n```"},
+        {"role": "assistant", "content": "On my way! !goToPlayer('zZZn98')"},
         {"role": "system", "content": "Code execution finished successfully."},
         {"role": "assistant", "content": "Here!"}
     ],
 
     [
         {"role": "user", "content": "hanky: collect some sand and give it to me please"},
-        {"role": "assistant", "content": "Collecting sand...\n```\n// I am going to collect 3 sand and give to hanky.\nawait skills.collectBlock(bot, 'sand', 3);\nawait skills.giveToPlayer(bot, 'sand', 'hanky');\n```"},
+        {"role": "assistant", "content": "Collecting sand... !newAction"},
         {"role": "system", "content": "Code Output:\nYou have reached player hanky.\nCode execution finished successfully."},
         {"role": "assistant", "content": "Here!"}
     ],
 
     [
         {"role": "user", "content": "joe: Follow me!"},
-        {"role": "assistant", "content": "Sure!\n```\n// I am going to follow joe.\nawait skills.followPlayer(bot, 'joe');\n```"}
+        {"role": "assistant", "content": "Sure! !followPlayer('joe')"}
     ],
 
     [
         {"role": "user", "content": "hanky: kill that zombie!"},
-        {"role": "assistant", "content": "I'm attacking!\n```\n//I'm going to attack the nearest zombie.\nawait skills.attackMob(bot, 'zombie');\n```"},
+        {"role": "assistant", "content": "I'm attacking! !attack('zombie')"},
         {"role": "system", "content": "Code Output:\nNo zombie nearby"},
         {"role": "assistant", "content": "I could not find a zombie nearby."}
     ],
 
     [
         {"role": "user", "content": "billybob: stop"},
-        {"role": "assistant", "content": "```\n// I am going to write empty code to stop whatever I am doing\n```"}
+        {"role": "assistant", "content": "Sure. !stop"}
     ],
 
     [
         {"role": "user", "content": "maya: go to the nearest oak log"},
-        {"role": "assistant", "content": "```\n// I am navigating to the nearest oak log.\nawait skills.goToPosition(bot, nearestOakLog.x, nearestOakLog.y, nearestOakLog.z);\n```"},
+        {"role": "assistant", "content": "!newAction`"},
         {"role": "system", "content": "Code output:\n!!Code threw exception!!  Error: ReferenceError: nearestOakLog is not defined\n Write code to fix the problem and try again."},
-        {"role": "assistant", "content": "Apologies, I made an error. Let me try that again.```\n// I am navigating to the nearest oak log.\nlet position = world.getNearestBlock(bot, 'oak_log', 20);\nawait skills.goToPosition(bot, position.x, position.y, position.z);\n```"}
+        {"role": "assistant", "content": "Apologies, I made an error. Would you like me to try again?"}
     ],
 
     [
         {"role": "user", "content": "trixy88: craft some sticks"},
-        {"role": "assistant", "content": "```\n// I am going to craft 8 sticks\nawait skills.craftRecipe(bot, 'stick');\nlog(bot, crafted)\nawait skills.craftRecipe(bot, 'stick');```"}
+        {"role": "assistant", "content": "!craftRecipe('stick', 4)"},
+        {"role": "system", "content": "Code Output:\nYou have crafted 16 sticks.\nCode execution finished successfully."},
+        {"role": "assistant", "content": "I've crafted 16 sticks!"}
     ]
 ]

src/examples_coder.json

@@ -0,0 +1,34 @@
+[
+    [
+        {"role": "user", "content": "greg: Collect 10 wood"},
+        {"role": "assistant", "content": "```await skills.collectBlock(bot, 'oak_log', 10);\n```"},
+        {"role": "system", "content": "Successfully collected 9 wood."},
+        {"role": "assistant", "content": "I collected 9 oak logs, what next?"}
+    ],
+    [
+        {"role": "user", "content": "bobby: Craft a plank"},
+        {"role": "assistant", "content": "```\nawait skills.craftRecipe(bot, 'spruce_planks');\n```"},
+        {"role": "system", "content": "Successfully crafted 4 spruce_planks."},        
+        {"role": "assistant", "content": "I crafted 4 spruce planks!"}
+    ],
+    [
+        {"role": "user", "content": "zZZn98: come here"},
+        {"role": "assistant", "content": "```await skills.goToPlayer(bot, 'zZZn98');\n```"},
+        {"role": "system", "content": "Code execution finished successfully."},
+        {"role": "assistant", "content": "Here!"}
+    ],
+    [
+        {"role": "user", "content": "maya: go to the nearest oak log"},
+        {"role": "assistant", "content": "```await skills.goToPosition(bot, nearestOakLog.x, nearestOakLog.y, nearestOakLog.z);\n```"},
+        {"role": "system", "content": "Code output:\n!!Code threw exception!!  Error: ReferenceError: nearestOakLog is not defined\n Write code to fix the problem and try again."},
+        {"role": "assistant", "content": "Apologies, I made an error. Let me try that again.```\n// I am navigating to the nearest oak log.\nlet position = world.getNearestBlock(bot, 'oak_log', 20);\nawait skills.goToPosition(bot, position.x, position.y, position.z);\n```"},
+        {"role": "system", "content": "Arrived at location."},
+        {"role": "assistant", "content": "I found an oak log and I am now standing next to it. What next?"}
+    ],
+    [
+        {"role": "user", "content": "234jeb: build a little tower"},
+        {"role": "assistant", "content": "```let pos = bot.entity.position;\nfor (let i = 0; i < 5; i++) {\n    await skills.placeBlock(bot, 'dirt', pos.x, pos.y + i, pos.z);\n}\n```"},
+        {"role": "system", "content": "Successfully placed 5 dirt."},
+        {"role": "assistant", "content": "I built a little tower!"}
+    ]
+]

src/process/init-agent.js

@@ -20,4 +20,4 @@ const argv = yargs(args)
     }).argv
 
 const name = args[0];
-new Agent(name, argv.profile, argv.init_message);
+new Agent().start(name, argv.profile, argv.init_message);

src/utils/mcdata.js

@@ -3,6 +3,9 @@ import { createBot } from 'mineflayer';
 import { pathfinder } from 'mineflayer-pathfinder';
 import { plugin as pvp } from 'mineflayer-pvp';
 import { plugin as collectblock } from 'mineflayer-collectblock';
+import { plugin as autoEat } from 'mineflayer-auto-eat';
+import plugin from 'mineflayer-armor-manager';
+const armorManager = plugin;
 
 const mc_version = '1.19.3'
 const mcdata = minecraftData(mc_version);
@@ -18,6 +21,9 @@ export function initBot(username) {
     bot.loadPlugin(pathfinder);
     bot.loadPlugin(pvp);
     bot.loadPlugin(collectblock);
+    bot.loadPlugin(autoEat);
+    bot.loadPlugin(armorManager); // auto equip armor
+
     return bot;
 }
 
@@ -77,3 +83,8 @@ export function getAllBlockIds(ignore) {
     }
     return blockIds;
 }
+
+export function getBiomeName(bot) {
+    const biomeId = bot.world.getBiome(bot.entity.position);
+    return mcdata.biomes[biomeId].name;
+}

src/utils/text.js

@@ -0,0 +1,14 @@
+export function stringifyTurns(turns) {
+    let res = '';
+    for (let turn of turns) {
+        if (turn.role === 'assistant') {
+            res += `\nYour output:\n${turn.content}`;
+        } else if (turn.role === 'system') {
+            res += `\nSystem output: ${turn.content}`;
+        } else {
+            res += `\nUser input: ${turn.content}`;
+        
+        }
+    }
+    return res.trim();
+}