From a14d4af65fafc4bfffc76a5e88bd1f18ae6f08fb Mon Sep 17 00:00:00 2001 From: Ayush Maniar Date: Sun, 5 Jan 2025 19:27:44 -0800 Subject: [PATCH 01/13] Add getCraftingPlan tool for making agents faster and resource efficient --- profiles/_default.json | 12 +++ src/agent/commands/actions.js | 2 +- src/agent/commands/queries.js | 39 ++++++++- src/agent/library/skills.js | 2 +- src/agent/npc/item_goal.js | 2 +- src/utils/mcdata.js | 157 +++++++++++++++++++++++++++++++++- 6 files changed, 209 insertions(+), 5 deletions(-) diff --git a/profiles/_default.json b/profiles/_default.json index 2a67725..cfcae6d 100644 --- a/profiles/_default.json +++ b/profiles/_default.json @@ -200,6 +200,18 @@ [ {"role": "system", "content": "You are self-prompting with the goal: 'Build a house'. Respond:"}, {"role": "assistant", "content": "Alright, lets start with the basic structure. !newAction(\"Build an 8x8 base for the house.\")"} + ], + [ + {"role": "user", "content": "trex56: Lets make a wooden pickaxe together"}, + {"role": "assistant", "content": "Sure, let me check how to make a wooden pickaxe !getCraftingPlan(\"wooden_pickaxe\",1)"}, + {"role": "system", "content": "You are missing the following items:\n- 2 oak_log\n\nOnce you have these items, here's your crafting plan:\n\nCraft 1 oak_log -> 4 oak_planks\nCraft 1 oak_log -> 4 oak_planks\nCraft 2 oak_planks -> 4 stick\nCraft 3 oak_planks + 2 stick -> 1 wooden_pickaxe"}, + {"role": "assistant", "content": "I need 2 oak logs to make a wooden pickaxe. Do you have any?"} + ], + [ + {"role": "user", "content": "terrance: (FROM OTHER BOT) I want you to craft 5 chains"}, + {"role": "assistant", "content": "Sure, first let me check how to build a chain, !getCraftingPlan(\"chain\",5)"}, + {"role": "system", "content": "You are missing the following items:\n- 7 iron_ingot\n\nOnce you have these items, here's your crafting plan:\n\nCraft 2 iron_ingot -> 18 iron_nugget\nCraft 10 iron_nugget + 5 iron_ingot -> 5 chain"}, + {"role": "assistant", "content": "Seems like I need 7 iron_ingots for 5 chains. Do you have any?"} ] ], diff --git a/src/agent/commands/actions.js b/src/agent/commands/actions.js index 6ee481b..9056d29 100644 --- a/src/agent/commands/actions.js +++ b/src/agent/commands/actions.js @@ -406,7 +406,7 @@ export const actionsList = [ convoManager.endConversation(player_name); return `Converstaion with ${player_name} ended.`; } - } + }, // { // commented for now, causes confusion with goal command // name: '!npcGoal', // description: 'Set a simple goal for an item or building to automatically work towards. Do not use for complex goals.', diff --git a/src/agent/commands/queries.js b/src/agent/commands/queries.js index b4dc74a..59a8416 100644 --- a/src/agent/commands/queries.js +++ b/src/agent/commands/queries.js @@ -176,5 +176,42 @@ export const queryList = [ perform: async function (agent) { return "Saved place names: " + agent.memory_bank.getKeys(); } - } + }, + { + name: '!getCraftingPlan', + description: "Provides a comprehensive crafting plan for a specified item. This includes a breakdown of required ingredients, the exact quantities needed, and an analysis of missing ingredients or extra items needed based on the bot's current inventory.", + params: { + targetItem: { + type: 'string', + description: 'The item that we are trying to craft' + }, + quantity: { + type: 'int', + description: 'The quantity of the item that we are trying to craft', + optional: true, + domain: [1, Infinity, '[)'], // Quantity must be at least 1, + default: 1 + } + }, + perform: function (agent, targetItem, quantity = 1) { + let bot = agent.bot; + + // Fetch the bot's inventory + const curr_inventory = world.getInventoryCounts(bot); + const target_item = targetItem; + let existingCount = curr_inventory[target_item] || 0; + var prefixMessage = ''; + if (existingCount > 0) { + curr_inventory[target_item] -= existingCount; + prefixMessage = `You already have ${existingCount} ${target_item} in your inventory. If you need to craft more,\n`; + } + + // Generate crafting plan + var craftingPlan = mc.getDetailedCraftingPlan(target_item, quantity, curr_inventory); + craftingPlan = prefixMessage + craftingPlan; + console.log('\n\n\n\n\n\n\n\n\n\n\n'); + console.log(craftingPlan); + return pad(craftingPlan); + } + }, ]; diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index 23f30ad..7293c0d 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -79,7 +79,7 @@ export async function craftRecipe(bot, itemName, num=1) { } } if (!recipes || recipes.length === 0) { - log(bot, `You do not have the resources to craft a ${itemName}. It requires: ${Object.entries(mc.getItemCraftingRecipes(itemName)[0]).map(([key, value]) => `${key}: ${value}`).join(', ')}.`); + log(bot, `You do not have the resources to craft a ${itemName}. It requires: ${Object.entries(mc.getItemCraftingRecipes(itemName)[0][0]).map(([key, value]) => `${key}: ${value}`).join(', ')}.`); if (placedTable) { await collectBlock(bot, 'crafting_table', 1); } diff --git a/src/agent/npc/item_goal.js b/src/agent/npc/item_goal.js index 40589ba..9055f54 100644 --- a/src/agent/npc/item_goal.js +++ b/src/agent/npc/item_goal.js @@ -204,7 +204,7 @@ class ItemWrapper { } createChildren() { - let recipes = mc.getItemCraftingRecipes(this.name); + let recipes = mc.getItemCraftingRecipes(this.name).map(([recipe, craftedCount]) => recipe); if (recipes) { for (let recipe of recipes) { let includes_blacklisted = false; diff --git a/src/utils/mcdata.js b/src/utils/mcdata.js index 58cfbdb..2a3a27c 100644 --- a/src/utils/mcdata.js +++ b/src/utils/mcdata.js @@ -190,7 +190,10 @@ export function getItemCraftingRecipes(itemName) { recipe[ingredientName] = 0; recipe[ingredientName]++; } - recipes.push(recipe); + recipes.push([ + recipe, + {craftedCount : r.result.count} + ]); } return recipes; @@ -327,4 +330,156 @@ export function calculateLimitingResource(availableItems, requiredItems, discret } if(discrete) num = Math.floor(num); return {num, limitingResource} +} + +let loopingItems = new Set(); + +export function initializeLoopingItems() { + + loopingItems = new Set(['coal', + 'wheat', + 'diamond', + 'emerald', + 'raw_iron', + 'raw_gold', + 'redstone', + 'blue_wool', + 'packed_mud', + 'raw_copper', + 'iron_ingot', + 'dried_kelp', + 'gold_ingot', + 'slime_ball', + 'black_wool', + 'quartz_slab', + 'copper_ingot', + 'lapis_lazuli', + 'honey_bottle', + 'rib_armor_trim_smithing_template', + 'eye_armor_trim_smithing_template', + 'vex_armor_trim_smithing_template', + 'dune_armor_trim_smithing_template', + 'host_armor_trim_smithing_template', + 'tide_armor_trim_smithing_template', + 'wild_armor_trim_smithing_template', + 'ward_armor_trim_smithing_template', + 'coast_armor_trim_smithing_template', + 'spire_armor_trim_smithing_template', + 'snout_armor_trim_smithing_template', + 'shaper_armor_trim_smithing_template', + 'netherite_upgrade_smithing_template', + 'raiser_armor_trim_smithing_template', + 'sentry_armor_trim_smithing_template', + 'silence_armor_trim_smithing_template', + 'wayfinder_armor_trim_smithing_template']); +} + + +/** + * Gets a detailed plan for crafting an item considering current inventory + */ +export function getDetailedCraftingPlan(targetItem, count = 1, current_inventory = {}) { + initializeLoopingItems(); + if (!targetItem || count <= 0 || !getItemId(targetItem)) { + return "Invalid input. Please provide a valid item name and positive count."; + } + + if (isBaseItem(targetItem)) { + const available = current_inventory[targetItem] || 0; + if (available >= count) return "You have all required items already in your inventory!"; + return `${targetItem} is a base item, you need to find ${count - available} more in the world`; + } + + const inventory = { ...current_inventory }; + const leftovers = {}; + const plan = craftItem(targetItem, count, inventory, leftovers); + return formatPlan(plan); +} + +function isBaseItem(item) { + return loopingItems.has(item) || getItemCraftingRecipes(item) === null; +} + +function craftItem(item, count, inventory, leftovers, crafted = { required: {}, steps: [], leftovers: {} }) { + // Check available inventory and leftovers first + const availableInv = inventory[item] || 0; + const availableLeft = leftovers[item] || 0; + const totalAvailable = availableInv + availableLeft; + + if (totalAvailable >= count) { + // Use leftovers first, then inventory + const useFromLeft = Math.min(availableLeft, count); + leftovers[item] = availableLeft - useFromLeft; + + const remainingNeeded = count - useFromLeft; + if (remainingNeeded > 0) { + inventory[item] = availableInv - remainingNeeded; + } + return crafted; + } + + // Use whatever is available + const stillNeeded = count - totalAvailable; + if (availableLeft > 0) leftovers[item] = 0; + if (availableInv > 0) inventory[item] = 0; + + if (isBaseItem(item)) { + crafted.required[item] = (crafted.required[item] || 0) + stillNeeded; + return crafted; + } + + const recipe = getItemCraftingRecipes(item)?.[0]; + if (!recipe) { + crafted.required[item] = stillNeeded; + return crafted; + } + + const [ingredients, result] = recipe; + const craftedPerRecipe = result.craftedCount; + const batchCount = Math.ceil(stillNeeded / craftedPerRecipe); + const totalProduced = batchCount * craftedPerRecipe; + + // Add excess to leftovers + if (totalProduced > stillNeeded) { + leftovers[item] = (leftovers[item] || 0) + (totalProduced - stillNeeded); + } + + // Process each ingredient + for (const [ingredientName, ingredientCount] of Object.entries(ingredients)) { + const totalIngredientNeeded = ingredientCount * batchCount; + craftItem(ingredientName, totalIngredientNeeded, inventory, leftovers, crafted); + } + + // Add crafting step + const stepIngredients = Object.entries(ingredients) + .map(([name, amount]) => `${amount * batchCount} ${name}`) + .join(' + '); + crafted.steps.push(`Craft ${stepIngredients} -> ${totalProduced} ${item}`); + + return crafted; +} + +function formatPlan({ required, steps, leftovers }) { + const lines = []; + + if (Object.keys(required).length > 0) { + lines.push('You are missing the following items:'); + Object.entries(required).forEach(([item, count]) => + lines.push(`- ${count} ${item}`)); + lines.push('\nOnce you have these items, here\'s your crafting plan:'); + } else { + lines.push('You have all items required to craft this item!'); + lines.push('Here\'s your crafting plan:'); + } + + lines.push(''); + lines.push(...steps); + + if (Object.keys(leftovers).length > 0) { + lines.push('\nYou will have leftover:'); + Object.entries(leftovers).forEach(([item, count]) => + lines.push(`- ${count} ${item}`)); + } + + return lines.join('\n'); } \ No newline at end of file From fbde286931661f36c21861e2977a8f83beee8645 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Fri, 24 Jan 2025 11:31:26 -0600 Subject: [PATCH 02/13] better auto-prompt --- src/agent/self_prompter.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/agent/self_prompter.js b/src/agent/self_prompter.js index 2c2f63c..439b6c6 100644 --- a/src/agent/self_prompter.js +++ b/src/agent/self_prompter.js @@ -38,7 +38,7 @@ export class SelfPrompter { let no_command_count = 0; const MAX_NO_COMMAND = 3; while (!this.interrupt) { - const msg = `You are self-prompting with the goal: '${this.prompt}'. Your next response MUST contain a command !withThisSyntax. Respond:`; + const msg = `You are self-prompting with the goal: '${this.prompt}'. Your next response MUST contain a command with this syntax: !commandName. Respond:`; let used_command = await this.agent.handleMessage('system', msg, -1); if (!used_command) { From 42f805cd160994af1734876f29e38601e707ffbc Mon Sep 17 00:00:00 2001 From: Pixel Date: Sat, 25 Jan 2025 10:53:22 -0600 Subject: [PATCH 03/13] Update README.md --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d165045..369cca2 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,11 @@ # Mindcraft 🧠⛏️ -Crafting minds for Minecraft with LLMs and Mineflayer! +Crafting minds for Minecraft with LLMs and [Mineflayer!](https://prismarinejs.github.io/mineflayer/#/) [FAQ](https://github.com/kolbytn/mindcraft/blob/main/FAQ.md) | [Discord Support](https://discord.gg/mp73p35dzC) | [Blog Post](https://kolbynottingham.com/mindcraft/) | [Contributor TODO](https://github.com/users/kolbytn/projects/1) -#### ‼️Warning‼️ - +> [!WARNING] Do not connect this bot to public servers with coding enabled. This project allows an LLM to write/execute code on your computer. While the code is sandboxed, it is still vulnerable to injection attacks on public servers. Code writing is disabled by default, you can enable it by setting `allow_insecure_coding` to `true` in `settings.js`. We strongly recommend running with additional layers of security such as docker containers. Ye be warned. ## Requirements @@ -63,7 +62,8 @@ To connect to online servers your bot will need an official Microsoft/Minecraft // rest is same... ``` -‼️ The bot's name in the profile.json must exactly match the Minecraft profile name! Otherwise the bot will spam talk to itself. +> [!WARNING] +> The bot's name in the profile.json must exactly match the Minecraft profile name! Otherwise the bot will spam talk to itself. To use different accounts, Mindcraft will connect with the account that the Minecraft launcher is currently using. You can switch accounts in the launcer, then run `node main.js`, then switch to your main account after the bot has connected. From 1e6ee45f0135a5194cefc11be39533ff3cb99b5a Mon Sep 17 00:00:00 2001 From: Pixel Date: Sat, 25 Jan 2025 10:55:39 -0600 Subject: [PATCH 04/13] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 369cca2..bf72fb6 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ To connect to online servers your bot will need an official Microsoft/Minecraft // rest is same... ``` -> [!WARNING] +> [!CAUTION] > The bot's name in the profile.json must exactly match the Minecraft profile name! Otherwise the bot will spam talk to itself. To use different accounts, Mindcraft will connect with the account that the Minecraft launcher is currently using. You can switch accounts in the launcer, then run `node main.js`, then switch to your main account after the bot has connected. From 063d42176447c0274fcaa9b1fe98091be3e2f137 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Sat, 25 Jan 2025 12:26:22 -0600 Subject: [PATCH 05/13] added second model for coding --- src/agent/prompter.js | 132 +++++++++++++++++++++++------------------- 1 file changed, 74 insertions(+), 58 deletions(-) diff --git a/src/agent/prompter.js b/src/agent/prompter.js index 310ca3e..5b09719 100644 --- a/src/agent/prompter.js +++ b/src/agent/prompter.js @@ -34,7 +34,6 @@ export class Prompter { this.coding_examples = null; let name = this.profile.name; - let chat = this.profile.model; this.cooldown = this.profile.cooldown ? this.profile.cooldown : 0; this.last_prompt_time = 0; this.awaiting_coding = false; @@ -43,68 +42,22 @@ export class Prompter { let max_tokens = null; if (this.profile.max_tokens) max_tokens = this.profile.max_tokens; - if (typeof chat === 'string' || chat instanceof String) { - chat = {model: chat}; - if (chat.model.includes('gemini')) - chat.api = 'google'; - else if (chat.model.includes('gpt') || chat.model.includes('o1')) - chat.api = 'openai'; - else if (chat.model.includes('claude')) - chat.api = 'anthropic'; - else if (chat.model.includes('huggingface/')) - chat.api = "huggingface"; - else if (chat.model.includes('meta/') || chat.model.includes('replicate/')) - chat.api = 'replicate'; - else if (chat.model.includes('mistralai/') || chat.model.includes("mistral/")) - chat.api = 'mistral'; - else if (chat.model.includes("groq/") || chat.model.includes("groqcloud/")) - chat.api = 'groq'; - else if (chat.model.includes('novita/')) - chat.api = 'novita'; - else if (chat.model.includes('qwen')) - chat.api = 'qwen'; - else if (chat.model.includes('grok')) - chat.api = 'xai'; - else if (chat.model.includes('deepseek')) - chat.api = 'deepseek'; - else - chat.api = 'ollama'; - } - console.log('Using chat settings:', chat); + let chat_model_profile = this._selectAPI(this.profile.model); + this.chat_model = this._createModel(chat_model_profile); - if (chat.api === 'google') - this.chat_model = new Gemini(chat.model, chat.url); - else if (chat.api === 'openai') - this.chat_model = new GPT(chat.model, chat.url); - else if (chat.api === 'anthropic') - this.chat_model = new Claude(chat.model, chat.url); - else if (chat.api === 'replicate') - this.chat_model = new ReplicateAPI(chat.model, chat.url); - else if (chat.api === 'ollama') - this.chat_model = new Local(chat.model, chat.url); - else if (chat.api === 'mistral') - this.chat_model = new Mistral(chat.model, chat.url); - else if (chat.api === 'groq') { - this.chat_model = new GroqCloudAPI(chat.model.replace('groq/', '').replace('groqcloud/', ''), chat.url, max_tokens ? max_tokens : 8192); + if (this.profile.code_model) { + let code_model_profile = this._selectAPI(this.profile.code_model); + this.code_model = this._createModel(code_model_profile); + } + else { + this.code_model = this.chat_model; } - else if (chat.api === 'huggingface') - this.chat_model = new HuggingFace(chat.model, chat.url); - else if (chat.api === 'novita') - this.chat_model = new Novita(chat.model.replace('novita/', ''), chat.url); - else if (chat.api === 'qwen') - this.chat_model = new Qwen(chat.model, chat.url); - else if (chat.api === 'xai') - this.chat_model = new Grok(chat.model, chat.url); - else if (chat.api === 'deepseek') - this.chat_model = new DeepSeek(chat.model, chat.url); - else - throw new Error('Unknown API:', api); let embedding = this.profile.embedding; if (embedding === undefined) { - if (chat.api !== 'ollama') - embedding = {api: chat.api}; + if (chat_model_profile.api !== 'ollama') + embedding = {api: chat_model_profile.api}; else embedding = {api: 'none'}; } @@ -146,6 +99,69 @@ export class Prompter { }); } + _selectAPI(profile) { + if (typeof profile === 'string' || profile instanceof String) { + profile = {model: profile}; + if (profile.model.includes('gemini')) + profile.api = 'google'; + else if (profile.model.includes('gpt') || profile.model.includes('o1')) + profile.api = 'openai'; + else if (profile.model.includes('claude')) + profile.api = 'anthropic'; + else if (profile.model.includes('huggingface/')) + profile.api = "huggingface"; + else if (profile.model.includes('meta/') || profile.model.includes('replicate/')) + profile.api = 'replicate'; + else if (profile.model.includes('mistralai/') || profile.model.includes("mistral/")) + model_profile.api = 'mistral'; + else if (profile.model.includes("groq/") || profile.model.includes("groqcloud/")) + profile.api = 'groq'; + else if (profile.model.includes('novita/')) + profile.api = 'novita'; + else if (profile.model.includes('qwen')) + profile.api = 'qwen'; + else if (profile.model.includes('grok')) + profile.api = 'xai'; + else if (profile.model.includes('deepseek')) + profile.api = 'deepseek'; + else + profile.api = 'ollama'; + } + return profile; + } + + _createModel(profile) { + let model = null; + if (profile.api === 'google') + model = new Gemini(profile.model, profile.url); + else if (profile.api === 'openai') + model = new GPT(profile.model, profile.url); + else if (profile.api === 'anthropic') + model = new Claude(profile.model, profile.url); + else if (profile.api === 'replicate') + model = new ReplicateAPI(profile.model, profile.url); + else if (profile.api === 'ollama') + model = new Local(profile.model, profile.url); + else if (profile.api === 'mistral') + model = new Mistral(profile.model, profile.url); + else if (profile.api === 'groq') { + model = new GroqCloudAPI(profile.model.replace('groq/', '').replace('groqcloud/', ''), profile.url, max_tokens ? max_tokens : 8192); + } + else if (profile.api === 'huggingface') + model = new HuggingFace(profile.model, profile.url); + else if (profile.api === 'novita') + model = new Novita(profile.model.replace('novita/', ''), profile.url); + else if (profile.api === 'qwen') + model = new Qwen(profile.model, profile.url); + else if (profile.api === 'xai') + model = new Grok(profile.model, profile.url); + else if (profile.api === 'deepseek') + model = new DeepSeek(profile.model, profile.url); + else + throw new Error('Unknown API:', api); + return model; + } + getName() { return this.profile.name; } @@ -273,7 +289,7 @@ export class Prompter { await this.checkCooldown(); let prompt = this.profile.coding; prompt = await this.replaceStrings(prompt, messages, this.coding_examples); - let resp = await this.chat_model.sendRequest(messages, prompt); + let resp = await this.code_model.sendRequest(messages, prompt); this.awaiting_coding = false; return resp; } From e1a9ed811b6690a60d6f14908e0ba1609c397e99 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Sat, 25 Jan 2025 12:26:49 -0600 Subject: [PATCH 06/13] small fix to block placing/farming --- src/agent/commands/index.js | 2 +- src/agent/library/skills.js | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/src/agent/commands/index.js b/src/agent/commands/index.js index 008c1d0..f40c5c2 100644 --- a/src/agent/commands/index.js +++ b/src/agent/commands/index.js @@ -160,7 +160,7 @@ export function parseCommandMessage(message) { suppressNoDomainWarning = true; //Don't spam console. Only give the warning once. } } else if(param.type === 'BlockName') { //Check that there is a block with this name - if(getBlockId(arg) == null) return `Invalid block type: ${arg}.` + if(getBlockId(arg) == null && arg !== 'air') return `Invalid block type: ${arg}.` } else if(param.type === 'ItemName') { //Check that there is an item with this name if(getItemId(arg) == null) return `Invalid item type: ${arg}.` } diff --git a/src/agent/library/skills.js b/src/agent/library/skills.js index 726ef18..78f1ad3 100644 --- a/src/agent/library/skills.js +++ b/src/agent/library/skills.js @@ -1275,8 +1275,14 @@ export async function tillAndSow(bot, x, y, z, seedType=null) { let block = bot.blockAt(new Vec3(x, y, z)); if (bot.modes.isOn('cheat')) { - placeBlock(bot, x, y, z, 'farmland'); - placeBlock(bot, x, y+1, z, seedType); + let to_remove = ['_seed', '_seeds']; + for (let remove of to_remove) { + if (seedType.endsWith(remove)) { + seedType = seedType.replace(remove, ''); + } + } + placeBlock(bot, 'farmland', x, y, z); + placeBlock(bot, seedType, x, y+1, z); return true; } From 9b387649a17b8a75c5f06e121053b1308180dd79 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Mon, 3 Feb 2025 18:35:58 -0600 Subject: [PATCH 07/13] enable o3, improve novita --- src/agent/prompter.js | 2 +- src/models/gpt.js | 2 +- src/models/novita.js | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/src/agent/prompter.js b/src/agent/prompter.js index 5b09719..b88001f 100644 --- a/src/agent/prompter.js +++ b/src/agent/prompter.js @@ -104,7 +104,7 @@ export class Prompter { profile = {model: profile}; if (profile.model.includes('gemini')) profile.api = 'google'; - else if (profile.model.includes('gpt') || profile.model.includes('o1')) + else if (profile.model.includes('gpt') || profile.model.includes('o1')|| profile.model.includes('o3')) profile.api = 'openai'; else if (profile.model.includes('claude')) profile.api = 'anthropic'; diff --git a/src/models/gpt.js b/src/models/gpt.js index dfd5e22..49be3a6 100644 --- a/src/models/gpt.js +++ b/src/models/gpt.js @@ -33,7 +33,7 @@ export class GPT { let res = null; try { - console.log('Awaiting openai api response...') + console.log('Awaiting openai api response from model', this.model_name) // console.log('Messages:', messages); let completion = await this.openai.chat.completions.create(pack); if (completion.choices[0].finish_reason == 'length') diff --git a/src/models/novita.js b/src/models/novita.js index d84aee7..33c05cc 100644 --- a/src/models/novita.js +++ b/src/models/novita.js @@ -1,5 +1,6 @@ import OpenAIApi from 'openai'; import { getKey } from '../utils/keys.js'; +import { strictFormat } from '../utils/text.js'; // llama, mistral export class Novita { @@ -17,6 +18,10 @@ export class Novita { async sendRequest(turns, systemMessage, stop_seq='***') { let messages = [{'role': 'system', 'content': systemMessage}].concat(turns); + + + messages = strictFormat(messages); + const pack = { model: this.model_name || "meta-llama/llama-3.1-70b-instruct", messages, @@ -41,6 +46,18 @@ export class Novita { res = 'My brain disconnected, try again.'; } } + if (res.includes('')) { + let start = res.indexOf(''); + let end = res.indexOf('') + 8; + if (start != -1) { + if (end != -1) { + res = res.substring(0, start) + res.substring(end); + } else { + res = res.substring(0, start+7); + } + } + res = res.trim(); + } return res; } From 23c54279ded2aaca6a33c2c83df788cc805470fe Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Mon, 3 Feb 2025 18:42:47 -0600 Subject: [PATCH 08/13] add code_model to readme --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d165045..1eaa1bd 100644 --- a/README.md +++ b/README.md @@ -120,7 +120,7 @@ LLM backends can be specified as simply as `"model": "gpt-3.5-turbo"`. However, } ``` -The model parameter accepts either a string or object. If a string, it should specify the model to be used. The api and url will be assumed. If an object, the api field must be specified. Each api has a default model and url, so those fields are optional. +The model or code_model parameter accepts either a string or object. If a string, it should specify the model to be used. The api and url will be assumed. If an object, the api field must be specified. Each api has a default model and url, so those fields are optional. If the embedding field is not specified, then it will use the default embedding method for the chat model's api (Note that anthropic has no embedding model). The embedding parameter can also be a string or object. If a string, it should specify the embedding api and the default model and url will be used. If a valid embedding is not specified and cannot be assumed, then word overlap will be used to retrieve examples instead. @@ -137,6 +137,7 @@ Thus, all the below specifications are equivalent to the above example: ```json "model": "gpt-3.5-turbo", "embedding": "openai" +"code_model": "gpt-3.5-turbo" ``` ## Patches From 60187e23171a58c8c991ddc5d48af24f2f415691 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Tue, 4 Feb 2025 13:02:57 -0600 Subject: [PATCH 09/13] added model parameters obj to profile --- README.md | 2 +- profiles/gpt.json | 8 +++- src/agent/agent.js | 10 ++--- src/models/claude.js | 10 +++-- src/models/deepseek.js | 4 +- src/models/gemini.js | 29 ++++++++++++-- src/models/gpt.js | 5 ++- src/models/grok.js | 7 +++- src/models/groq.js | 14 ++++--- src/models/huggingface.js | 6 ++- src/models/local.js | 10 ++++- src/models/mistral.js | 8 ++-- src/models/novita.js | 5 ++- src/{agent => models}/prompter.js | 63 ++++++++++++++++--------------- src/models/qwen.js | 11 ++++-- src/models/replicate.js | 9 ++++- src/process/init_agent.js | 7 +--- 17 files changed, 134 insertions(+), 74 deletions(-) rename src/{agent => models}/prompter.js (88%) diff --git a/README.md b/README.md index 1eaa1bd..d18fe65 100644 --- a/README.md +++ b/README.md @@ -42,7 +42,7 @@ You can configure the agent's name, model, and prompts in their profile like `an | OpenAI | `OPENAI_API_KEY` | `gpt-4o-mini` | [docs](https://platform.openai.com/docs/models) | | Google | `GEMINI_API_KEY` | `gemini-pro` | [docs](https://ai.google.dev/gemini-api/docs/models/gemini) | | Anthropic | `ANTHROPIC_API_KEY` | `claude-3-haiku-20240307` | [docs](https://docs.anthropic.com/claude/docs/models-overview) | -| Replicate | `REPLICATE_API_KEY` | `meta/meta-llama-3-70b-instruct` | [docs](https://replicate.com/collections/language-models) | +| Replicate | `REPLICATE_API_KEY` | `replicate/meta/meta-llama-3-70b-instruct` | [docs](https://replicate.com/collections/language-models) | | Ollama (local) | n/a | `llama3` | [docs](https://ollama.com/library) | | Groq | `GROQCLOUD_API_KEY` | `groq/mixtral-8x7b-32768` | [docs](https://console.groq.com/docs/models) | | Hugging Face | `HUGGINGFACE_API_KEY` | `huggingface/mistralai/Mistral-Nemo-Instruct-2407` | [docs](https://huggingface.co/models) | diff --git a/profiles/gpt.json b/profiles/gpt.json index 32d99c1..a5effe1 100644 --- a/profiles/gpt.json +++ b/profiles/gpt.json @@ -1,5 +1,11 @@ { "name": "gpt", - "model": "gpt-4o" + "model": { + "model": "gpt-4o-mini", + "params": { + "temperature": 1, + "not_real": true + } + } } \ No newline at end of file diff --git a/src/agent/agent.js b/src/agent/agent.js index 4691079..8e211ef 100644 --- a/src/agent/agent.js +++ b/src/agent/agent.js @@ -1,6 +1,6 @@ import { History } from './history.js'; import { Coder } from './coder.js'; -import { Prompter } from './prompter.js'; +import { Prompter } from '../models/prompter.js'; import { initModes } from './modes.js'; import { initBot } from '../utils/mcdata.js'; import { containsCommand, commandExists, executeCommand, truncCommandMessage, isAction, blacklistCommands } from './commands/index.js'; @@ -100,11 +100,9 @@ export class Agent { }); } catch (error) { // Ensure we're not losing error details - console.error('Agent start failed with error:', { - message: error.message || 'No error message', - stack: error.stack || 'No stack trace', - error: error - }); + console.error('Agent start failed with error') + console.error(error) + throw error; // Re-throw with preserved details } } diff --git a/src/models/claude.js b/src/models/claude.js index c97ecb2..f8d2a90 100644 --- a/src/models/claude.js +++ b/src/models/claude.js @@ -3,8 +3,9 @@ import { strictFormat } from '../utils/text.js'; import { getKey } from '../utils/keys.js'; export class Claude { - constructor(model_name, url) { + constructor(model_name, url, params) { this.model_name = model_name; + this.params = params; let config = {}; if (url) @@ -20,13 +21,16 @@ export class Claude { let res = null; try { console.log('Awaiting anthropic api response...') - // console.log('Messages:', messages); + if (!this.params.max_tokens) { + this.params.max_tokens = 4096; + } const resp = await this.anthropic.messages.create({ model: this.model_name || "claude-3-sonnet-20240229", system: systemMessage, - max_tokens: 2048, messages: messages, + ...(this.params || {}) }); + console.log('Received.') res = resp.content[0].text; } diff --git a/src/models/deepseek.js b/src/models/deepseek.js index 395aa8c..da98ba2 100644 --- a/src/models/deepseek.js +++ b/src/models/deepseek.js @@ -3,8 +3,9 @@ import { getKey, hasKey } from '../utils/keys.js'; import { strictFormat } from '../utils/text.js'; export class DeepSeek { - constructor(model_name, url) { + constructor(model_name, url, params) { this.model_name = model_name; + this.params = params; let config = {}; @@ -23,6 +24,7 @@ export class DeepSeek { model: this.model_name || "deepseek-chat", messages, stop: stop_seq, + ...(this.params || {}) }; let res = null; diff --git a/src/models/gemini.js b/src/models/gemini.js index 1536d66..de71a66 100644 --- a/src/models/gemini.js +++ b/src/models/gemini.js @@ -3,8 +3,9 @@ import { toSinglePrompt } from '../utils/text.js'; import { getKey } from '../utils/keys.js'; export class Gemini { - constructor(model_name, url) { + constructor(model_name, url, params) { this.model_name = model_name; + this.params = params; this.url = url; this.safetySettings = [ { @@ -34,15 +35,20 @@ export class Gemini { async sendRequest(turns, systemMessage) { let model; + const modelConfig = { + model: this.model_name || "gemini-1.5-flash", + ...(this.params || {}) + }; + if (this.url) { model = this.genAI.getGenerativeModel( - { model: this.model_name || "gemini-1.5-flash" }, + modelConfig, { baseUrl: this.url }, { safetySettings: this.safetySettings } ); } else { model = this.genAI.getGenerativeModel( - { model: this.model_name || "gemini-1.5-flash" }, + modelConfig, { safetySettings: this.safetySettings } ); } @@ -50,12 +56,27 @@ export class Gemini { const stop_seq = '***'; const prompt = toSinglePrompt(turns, systemMessage, stop_seq, 'model'); console.log('Awaiting Google API response...'); - const result = await model.generateContent(prompt); + const result = await model.generateContent({ + contents: [ + { + role: 'user', + parts: [ + { + text: "Explain how AI works", + } + ], + } + ], + generateConfig: { + ...(this.params || {}) + } + }); const response = await result.response; const text = response.text(); console.log('Received.'); if (!text.includes(stop_seq)) return text; const idx = text.indexOf(stop_seq); + return text.slice(0, idx); } diff --git a/src/models/gpt.js b/src/models/gpt.js index 49be3a6..1a88bf4 100644 --- a/src/models/gpt.js +++ b/src/models/gpt.js @@ -3,8 +3,9 @@ import { getKey, hasKey } from '../utils/keys.js'; import { strictFormat } from '../utils/text.js'; export class GPT { - constructor(model_name, url) { + constructor(model_name, url, params) { this.model_name = model_name; + this.params = params; let config = {}; if (url) @@ -25,6 +26,7 @@ export class GPT { model: this.model_name || "gpt-3.5-turbo", messages, stop: stop_seq, + ...(this.params || {}) }; if (this.model_name.includes('o1')) { pack.messages = strictFormat(messages); @@ -32,6 +34,7 @@ export class GPT { } let res = null; + try { console.log('Awaiting openai api response from model', this.model_name) // console.log('Messages:', messages); diff --git a/src/models/grok.js b/src/models/grok.js index 19a3b38..a8c6672 100644 --- a/src/models/grok.js +++ b/src/models/grok.js @@ -3,8 +3,10 @@ import { getKey } from '../utils/keys.js'; // xAI doesn't supply a SDK for their models, but fully supports OpenAI and Anthropic SDKs export class Grok { - constructor(model_name, url) { + constructor(model_name, url, params) { this.model_name = model_name; + this.url = url; + this.params = params; let config = {}; if (url) @@ -23,7 +25,8 @@ export class Grok { const pack = { model: this.model_name || "grok-beta", messages, - stop: [stop_seq] + stop: [stop_seq], + ...(this.params || {}) }; let res = null; diff --git a/src/models/groq.js b/src/models/groq.js index e17f13d..6911534 100644 --- a/src/models/groq.js +++ b/src/models/groq.js @@ -4,12 +4,13 @@ import { getKey } from '../utils/keys.js'; // Umbrella class for Mixtral, LLama, Gemma... export class GroqCloudAPI { - constructor(model_name, url, max_tokens=16384) { + constructor(model_name, url, params) { this.model_name = model_name; this.url = url; - this.max_tokens = max_tokens; + this.params = params; // ReplicateAPI theft :3 if (this.url) { + console.warn("Groq Cloud has no implementation for custom URLs. Ignoring provided URL."); } this.groq = new Groq({ apiKey: getKey('GROQCLOUD_API_KEY') }); @@ -20,14 +21,15 @@ export class GroqCloudAPI { let res = null; try { console.log("Awaiting Groq response..."); + if (!this.params.max_tokens) { + this.params.max_tokens = 16384; + } let completion = await this.groq.chat.completions.create({ "messages": messages, "model": this.model_name || "mixtral-8x7b-32768", - "temperature": 0.2, - "max_tokens": this.max_tokens, // maximum token limit, differs from model to model - "top_p": 1, "stream": true, - "stop": stop_seq // "***" + "stop": stop_seq, + ...(this.params || {}) }); let temp_res = ""; diff --git a/src/models/huggingface.js b/src/models/huggingface.js index 56f9d55..dd5c89d 100644 --- a/src/models/huggingface.js +++ b/src/models/huggingface.js @@ -3,9 +3,10 @@ import {getKey} from '../utils/keys.js'; import {HfInference} from "@huggingface/inference"; export class HuggingFace { - constructor(model_name, url) { + constructor(model_name, url, params) { this.model_name = model_name.replace('huggingface/',''); this.url = url; + this.params = params; if (this.url) { console.warn("Hugging Face doesn't support custom urls!"); @@ -25,7 +26,8 @@ export class HuggingFace { console.log('Awaiting Hugging Face API response...'); for await (const chunk of this.huggingface.chatCompletionStream({ model: model_name, - messages: [{ role: "user", content: input }] + messages: [{ role: "user", content: input }], + ...(this.params || {}) })) { res += (chunk.choices[0]?.delta?.content || ""); } diff --git a/src/models/local.js b/src/models/local.js index 18d06e0..23d7e0e 100644 --- a/src/models/local.js +++ b/src/models/local.js @@ -1,8 +1,9 @@ import { strictFormat } from '../utils/text.js'; export class Local { - constructor(model_name, url) { + constructor(model_name, url, params) { this.model_name = model_name; + this.params = params; this.url = url || 'http://127.0.0.1:11434'; this.chat_endpoint = '/api/chat'; this.embedding_endpoint = '/api/embeddings'; @@ -15,7 +16,12 @@ export class Local { let res = null; try { console.log(`Awaiting local response... (model: ${model})`) - res = await this.send(this.chat_endpoint, {model: model, messages: messages, stream: false}); + res = await this.send(this.chat_endpoint, { + model: model, + messages: messages, + stream: false, + ...(this.params || {}) + }); if (res) res = res['message']['content']; } diff --git a/src/models/mistral.js b/src/models/mistral.js index 3b41f78..b33d1de 100644 --- a/src/models/mistral.js +++ b/src/models/mistral.js @@ -5,10 +5,13 @@ import { strictFormat } from '../utils/text.js'; export class Mistral { #client; - constructor(model_name, url) { + constructor(model_name, url, params) { + this.model_name = model_name; + this.params = params; if (typeof url === "string") { console.warn("Mistral does not support custom URL's, ignoring!"); + } if (!getKey("MISTRAL_API_KEY")) { @@ -22,8 +25,6 @@ export class Mistral { ); - this.model_name = model_name; - // Prevents the following code from running when model not specified if (typeof this.model_name === "undefined") return; @@ -49,6 +50,7 @@ export class Mistral { const response = await this.#client.chat.complete({ model, messages, + ...(this.params || {}) }); result = response.choices[0].message.content; diff --git a/src/models/novita.js b/src/models/novita.js index 33c05cc..8f2dd08 100644 --- a/src/models/novita.js +++ b/src/models/novita.js @@ -4,9 +4,11 @@ import { strictFormat } from '../utils/text.js'; // llama, mistral export class Novita { - constructor(model_name, url) { + constructor(model_name, url, params) { this.model_name = model_name.replace('novita/', ''); this.url = url || 'https://api.novita.ai/v3/openai'; + this.params = params; + let config = { baseURL: this.url @@ -26,6 +28,7 @@ export class Novita { model: this.model_name || "meta-llama/llama-3.1-70b-instruct", messages, stop: [stop_seq], + ...(this.params || {}) }; let res = null; diff --git a/src/agent/prompter.js b/src/models/prompter.js similarity index 88% rename from src/agent/prompter.js rename to src/models/prompter.js index b88001f..91fcf3f 100644 --- a/src/agent/prompter.js +++ b/src/models/prompter.js @@ -1,23 +1,23 @@ import { readFileSync, mkdirSync, writeFileSync} from 'fs'; import { Examples } from '../utils/examples.js'; -import { getCommandDocs } from './commands/index.js'; -import { getSkillDocs } from './library/index.js'; +import { getCommandDocs } from '../agent/commands/index.js'; +import { getSkillDocs } from '../agent/library/index.js'; import { stringifyTurns } from '../utils/text.js'; -import { getCommand } from './commands/index.js'; +import { getCommand } from '../agent/commands/index.js'; import settings from '../../settings.js'; -import { Gemini } from '../models/gemini.js'; -import { GPT } from '../models/gpt.js'; -import { Claude } from '../models/claude.js'; -import { Mistral } from '../models/mistral.js'; -import { ReplicateAPI } from '../models/replicate.js'; -import { Local } from '../models/local.js'; -import { Novita } from '../models/novita.js'; -import { GroqCloudAPI } from '../models/groq.js'; -import { HuggingFace } from '../models/huggingface.js'; -import { Qwen } from "../models/qwen.js"; -import { Grok } from "../models/grok.js"; -import { DeepSeek } from '../models/deepseek.js'; +import { Gemini } from './gemini.js'; +import { GPT } from './gpt.js'; +import { Claude } from './claude.js'; +import { Mistral } from './mistral.js'; +import { ReplicateAPI } from './replicate.js'; +import { Local } from './local.js'; +import { Novita } from './novita.js'; +import { GroqCloudAPI } from './groq.js'; +import { HuggingFace } from './huggingface.js'; +import { Qwen } from "./qwen.js"; +import { Grok } from "./grok.js"; +import { DeepSeek } from './deepseek.js'; export class Prompter { constructor(agent, fp) { @@ -102,6 +102,8 @@ export class Prompter { _selectAPI(profile) { if (typeof profile === 'string' || profile instanceof String) { profile = {model: profile}; + } + if (!profile.api) { if (profile.model.includes('gemini')) profile.api = 'google'; else if (profile.model.includes('gpt') || profile.model.includes('o1')|| profile.model.includes('o3')) @@ -110,7 +112,7 @@ export class Prompter { profile.api = 'anthropic'; else if (profile.model.includes('huggingface/')) profile.api = "huggingface"; - else if (profile.model.includes('meta/') || profile.model.includes('replicate/')) + else if (profile.model.includes('replicate/')) profile.api = 'replicate'; else if (profile.model.includes('mistralai/') || profile.model.includes("mistral/")) model_profile.api = 'mistral'; @@ -133,32 +135,31 @@ export class Prompter { _createModel(profile) { let model = null; if (profile.api === 'google') - model = new Gemini(profile.model, profile.url); + model = new Gemini(profile.model, profile.url, profile.params); else if (profile.api === 'openai') - model = new GPT(profile.model, profile.url); + model = new GPT(profile.model, profile.url, profile.params); else if (profile.api === 'anthropic') - model = new Claude(profile.model, profile.url); + model = new Claude(profile.model, profile.url, profile.params); else if (profile.api === 'replicate') - model = new ReplicateAPI(profile.model, profile.url); + model = new ReplicateAPI(profile.model, profile.url, profile.params); else if (profile.api === 'ollama') - model = new Local(profile.model, profile.url); + model = new Local(profile.model, profile.url, profile.params); else if (profile.api === 'mistral') - model = new Mistral(profile.model, profile.url); - else if (profile.api === 'groq') { - model = new GroqCloudAPI(profile.model.replace('groq/', '').replace('groqcloud/', ''), profile.url, max_tokens ? max_tokens : 8192); - } + model = new Mistral(profile.model, profile.url, profile.params); + else if (profile.api === 'groq') + model = new GroqCloudAPI(profile.model.replace('groq/', '').replace('groqcloud/', ''), profile.url, profile.params); else if (profile.api === 'huggingface') - model = new HuggingFace(profile.model, profile.url); + model = new HuggingFace(profile.model, profile.url, profile.params); else if (profile.api === 'novita') - model = new Novita(profile.model.replace('novita/', ''), profile.url); + model = new Novita(profile.model.replace('novita/', ''), profile.url, profile.params); else if (profile.api === 'qwen') - model = new Qwen(profile.model, profile.url); + model = new Qwen(profile.model, profile.url, profile.params); else if (profile.api === 'xai') - model = new Grok(profile.model, profile.url); + model = new Grok(profile.model, profile.url, profile.params); else if (profile.api === 'deepseek') - model = new DeepSeek(profile.model, profile.url); + model = new DeepSeek(profile.model, profile.url, profile.params); else - throw new Error('Unknown API:', api); + throw new Error('Unknown API:', profile.api); return model; } diff --git a/src/models/qwen.js b/src/models/qwen.js index d3d7bec..5f3eafa 100644 --- a/src/models/qwen.js +++ b/src/models/qwen.js @@ -4,8 +4,9 @@ import { getKey } from '../utils/keys.js'; export class Qwen { - constructor(modelName, url) { - this.modelName = modelName; + constructor(model_name, url, params) { + this.model_name = model_name; + this.params = params; this.url = url || 'https://dashscope.aliyuncs.com/api/v1/services/aigc/text-generation/generation'; this.apiKey = getKey('QWEN_API_KEY'); } @@ -19,7 +20,11 @@ export class Qwen { const data = { model: this.modelName || 'qwen-plus', input: { messages: [{ role: 'system', content: systemMessage }, ...turns] }, - parameters: { result_format: 'message', stop: stopSeq }, + parameters: { + result_format: 'message', + stop: stopSeq, + ...(this.params || {}) + }, }; // Add default user message if all messages are 'system' role diff --git a/src/models/replicate.js b/src/models/replicate.js index e0c7d6c..c8c3ba3 100644 --- a/src/models/replicate.js +++ b/src/models/replicate.js @@ -4,9 +4,10 @@ import { getKey } from '../utils/keys.js'; // llama, mistral export class ReplicateAPI { - constructor(model_name, url) { + constructor(model_name, url, params) { this.model_name = model_name; this.url = url; + this.params = params; if (this.url) { console.warn('Replicate API does not support custom URLs. Ignoring provided URL.'); @@ -22,7 +23,11 @@ export class ReplicateAPI { const prompt = toSinglePrompt(turns, null, stop_seq); let model_name = this.model_name || 'meta/meta-llama-3-70b-instruct'; - const input = { prompt, system_prompt: systemMessage }; + const input = { + prompt, + system_prompt: systemMessage, + ...(this.params || {}) + }; let res = null; try { console.log('Awaiting Replicate API response...'); diff --git a/src/process/init_agent.js b/src/process/init_agent.js index 88c99b9..15b08e0 100644 --- a/src/process/init_agent.js +++ b/src/process/init_agent.js @@ -57,11 +57,8 @@ const argv = yargs(args) const agent = new Agent(); await agent.start(argv.profile, argv.load_memory, argv.init_message, argv.count_id, argv.task_path, argv.task_id); } catch (error) { - console.error('Failed to start agent process:', { - message: error.message || 'No error message', - stack: error.stack || 'No stack trace', - error: error - }); + console.error('Failed to start agent process:'); + console.error(error); process.exit(1); } })(); From be780cba27bb77bf44f5dbdd875778d1007fa7d3 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Tue, 4 Feb 2025 13:28:32 -0600 Subject: [PATCH 10/13] remove obsolete collectblock patche --- patches/mineflayer-collectblock+1.4.1.patch | 35 --------------------- 1 file changed, 35 deletions(-) delete mode 100644 patches/mineflayer-collectblock+1.4.1.patch diff --git a/patches/mineflayer-collectblock+1.4.1.patch b/patches/mineflayer-collectblock+1.4.1.patch deleted file mode 100644 index 1df504b..0000000 --- a/patches/mineflayer-collectblock+1.4.1.patch +++ /dev/null @@ -1,35 +0,0 @@ -diff --git a/node_modules/mineflayer-collectblock/lib/CollectBlock.js b/node_modules/mineflayer-collectblock/lib/CollectBlock.js -index 2c11e8c..bb49c11 100644 ---- a/node_modules/mineflayer-collectblock/lib/CollectBlock.js -+++ b/node_modules/mineflayer-collectblock/lib/CollectBlock.js -@@ -77,10 +77,11 @@ function mineBlock(bot, block, options) { - } - yield bot.tool.equipForBlock(block, equipToolOptions); - // @ts-expect-error -- if (!block.canHarvest(bot.heldItem)) { -+ if (bot.heldItem !== null && !block.canHarvest(bot.heldItem.type)) { - options.targets.removeTarget(block); - return; - } -+ - const tempEvents = new TemporarySubscriber_1.TemporarySubscriber(bot); - tempEvents.subscribeTo('itemDrop', (entity) => { - if (entity.position.distanceTo(block.position.offset(0.5, 0.5, 0.5)) <= 0.5) { -@@ -92,7 +93,7 @@ function mineBlock(bot, block, options) { - // Waiting for items to drop - yield new Promise(resolve => { - let remainingTicks = 10; -- tempEvents.subscribeTo('physicTick', () => { -+ tempEvents.subscribeTo('physicsTick', () => { - remainingTicks--; - if (remainingTicks <= 0) { - tempEvents.cleanup(); -@@ -195,6 +196,8 @@ class CollectBlock { - throw (0, Util_1.error)('UnresolvedDependency', 'The mineflayer-collectblock plugin relies on the mineflayer-tool plugin to run!'); - } - if (this.movements != null) { -+ this.movements.dontMineUnderFallingBlock = false; -+ this.movements.dontCreateFlow = false; - this.bot.pathfinder.setMovements(this.movements); - } - if (!optionsFull.append) From 0c3ba9a3830673e9ebe3c91ed262ab2421189dfb Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Tue, 4 Feb 2025 14:41:57 -0600 Subject: [PATCH 11/13] updated gemini, cleaned gpt profile --- profiles/gpt.json | 5 ++--- src/models/gemini.js | 32 +++++++++++++++----------------- src/utils/text.js | 6 ++++-- 3 files changed, 21 insertions(+), 22 deletions(-) diff --git a/profiles/gpt.json b/profiles/gpt.json index a5effe1..ad7097e 100644 --- a/profiles/gpt.json +++ b/profiles/gpt.json @@ -2,10 +2,9 @@ "name": "gpt", "model": { - "model": "gpt-4o-mini", + "model": "gpt-4o", "params": { - "temperature": 1, - "not_real": true + "temperature": 0.5 } } } \ No newline at end of file diff --git a/src/models/gemini.js b/src/models/gemini.js index de71a66..bcc1895 100644 --- a/src/models/gemini.js +++ b/src/models/gemini.js @@ -1,5 +1,5 @@ import { GoogleGenerativeAI } from '@google/generative-ai'; -import { toSinglePrompt } from '../utils/text.js'; +import { toSinglePrompt, strictFormat } from '../utils/text.js'; import { getKey } from '../utils/keys.js'; export class Gemini { @@ -37,7 +37,7 @@ export class Gemini { let model; const modelConfig = { model: this.model_name || "gemini-1.5-flash", - ...(this.params || {}) + // systemInstruction does not work bc google is trash }; if (this.url) { @@ -53,29 +53,27 @@ export class Gemini { ); } - const stop_seq = '***'; - const prompt = toSinglePrompt(turns, systemMessage, stop_seq, 'model'); console.log('Awaiting Google API response...'); + + turns.unshift({ role: 'system', content: systemMessage }); + turns = strictFormat(turns); + let contents = []; + for (let turn of turns) { + contents.push({ + role: turn.role === 'assistant' ? 'model' : 'user', + parts: [{ text: turn.content }] + }); + } + const result = await model.generateContent({ - contents: [ - { - role: 'user', - parts: [ - { - text: "Explain how AI works", - } - ], - } - ], - generateConfig: { + contents, + generationConfig: { ...(this.params || {}) } }); const response = await result.response; const text = response.text(); console.log('Received.'); - if (!text.includes(stop_seq)) return text; - const idx = text.indexOf(stop_seq); return text.slice(0, idx); } diff --git a/src/utils/text.js b/src/utils/text.js index 1e93667..f500199 100644 --- a/src/utils/text.js +++ b/src/utils/text.js @@ -26,8 +26,10 @@ export function toSinglePrompt(turns, system=null, stop_seq='***', model_nicknam return prompt; } -// ensures stricter turn order for anthropic/llama models -// combines repeated messages from the same role, separates repeat assistant messages with filler user messages +// ensures stricter turn order and roles: +// - system messages are treated as user messages and prefixed with SYSTEM: +// - combines repeated messages from users +// - separates repeat assistant messages with filler user messages export function strictFormat(turns) { let prev_role = null; let messages = []; From 402e09f03795036b8b49fbdde10c90f479f64b35 Mon Sep 17 00:00:00 2001 From: Max Robinson Date: Tue, 4 Feb 2025 15:00:55 -0600 Subject: [PATCH 12/13] better readme --- README.md | 35 ++++++++++++++++------------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index d18fe65..e735412 100644 --- a/README.md +++ b/README.md @@ -105,40 +105,37 @@ node main.js --profiles ./profiles/andy.json ./profiles/jill.json ### Model Specifications -LLM backends can be specified as simply as `"model": "gpt-3.5-turbo"`. However, for both the chat model and the embedding model, the bot profile can specify the below attributes: +LLM models can be specified as simply as `"model": "gpt-4o"`. However, you can specify different models for chat, coding, and embeddings. +You can pass a string or an object for these fields. A model object must specify an `api`, and optionally a `model`, `url`, and additional `params`. ```json "model": { "api": "openai", + "model": "gpt-4o", "url": "https://api.openai.com/v1/", - "model": "gpt-3.5-turbo" + "params": { + "max_tokens": 1000, + "temperature": 1 + } +}, +"code_model": { + "api": "openai", + "model": "gpt-4", + "url": "https://api.openai.com/v1/" }, "embedding": { "api": "openai", "url": "https://api.openai.com/v1/", "model": "text-embedding-ada-002" } + ``` -The model or code_model parameter accepts either a string or object. If a string, it should specify the model to be used. The api and url will be assumed. If an object, the api field must be specified. Each api has a default model and url, so those fields are optional. +`model` is used for chat, `code_model` is used for newAction coding, and `embedding` is used to embed text for example selection. If `code_model` is not specified, then it will use `model` for coding. -If the embedding field is not specified, then it will use the default embedding method for the chat model's api (Note that anthropic has no embedding model). The embedding parameter can also be a string or object. If a string, it should specify the embedding api and the default model and url will be used. If a valid embedding is not specified and cannot be assumed, then word overlap will be used to retrieve examples instead. +All apis have default models and urls, so those fields are optional. Note some apis have no embedding model, so they will default to word overlap to retrieve examples. -Thus, all the below specifications are equivalent to the above example: - -```json -"model": "gpt-3.5-turbo" -``` -```json -"model": { - "api": "openai" -} -``` -```json -"model": "gpt-3.5-turbo", -"embedding": "openai" -"code_model": "gpt-3.5-turbo" -``` +The `params` field is optional and can be used to specify additional parameters for the model. It accepts any key-value pairs supported by the api. Is not supported for embedding models. ## Patches From 9c675c51bcc042c70e4b6129857287ec593cccf7 Mon Sep 17 00:00:00 2001 From: MaxRobinsonTheGreat Date: Tue, 4 Feb 2025 16:26:00 -0600 Subject: [PATCH 13/13] fix syntax error, var->let, remove examples --- profiles/_default.json | 12 ------------ src/agent/commands/queries.js | 6 +++--- src/agent/tasks.js | 4 ++-- viewer.html | 12 ++++++------ 4 files changed, 11 insertions(+), 23 deletions(-) diff --git a/profiles/_default.json b/profiles/_default.json index edd56a4..ac244fc 100644 --- a/profiles/_default.json +++ b/profiles/_default.json @@ -200,18 +200,6 @@ [ {"role": "system", "content": "You are self-prompting with the goal: 'Build a house'. Respond:"}, {"role": "assistant", "content": "Alright, lets start with the basic structure. !newAction(\"Build an 8x8 base for the house.\")"} - ], - [ - {"role": "user", "content": "trex56: Lets make a wooden pickaxe together"}, - {"role": "assistant", "content": "Sure, let me check how to make a wooden pickaxe !getCraftingPlan(\"wooden_pickaxe\",1)"}, - {"role": "system", "content": "You are missing the following items:\n- 2 oak_log\n\nOnce you have these items, here's your crafting plan:\n\nCraft 1 oak_log -> 4 oak_planks\nCraft 1 oak_log -> 4 oak_planks\nCraft 2 oak_planks -> 4 stick\nCraft 3 oak_planks + 2 stick -> 1 wooden_pickaxe"}, - {"role": "assistant", "content": "I need 2 oak logs to make a wooden pickaxe. Do you have any?"} - ], - [ - {"role": "user", "content": "terrance: (FROM OTHER BOT) I want you to craft 5 chains"}, - {"role": "assistant", "content": "Sure, first let me check how to build a chain, !getCraftingPlan(\"chain\",5)"}, - {"role": "system", "content": "You are missing the following items:\n- 7 iron_ingot\n\nOnce you have these items, here's your crafting plan:\n\nCraft 2 iron_ingot -> 18 iron_nugget\nCraft 10 iron_nugget + 5 iron_ingot -> 5 chain"}, - {"role": "assistant", "content": "Seems like I need 7 iron_ingots for 5 chains. Do you have any?"} ] ], diff --git a/src/agent/commands/queries.js b/src/agent/commands/queries.js index 60049d0..c9fe333 100644 --- a/src/agent/commands/queries.js +++ b/src/agent/commands/queries.js @@ -201,18 +201,18 @@ export const queryList = [ const curr_inventory = world.getInventoryCounts(bot); const target_item = targetItem; let existingCount = curr_inventory[target_item] || 0; - var prefixMessage = ''; + let prefixMessage = ''; if (existingCount > 0) { curr_inventory[target_item] -= existingCount; prefixMessage = `You already have ${existingCount} ${target_item} in your inventory. If you need to craft more,\n`; } // Generate crafting plan - var craftingPlan = mc.getDetailedCraftingPlan(target_item, quantity, curr_inventory); + let craftingPlan = mc.getDetailedCraftingPlan(target_item, quantity, curr_inventory); craftingPlan = prefixMessage + craftingPlan; - console.log('\n\n\n\n\n\n\n\n\n\n\n'); console.log(craftingPlan); return pad(craftingPlan); + }, }, { name: '!help', diff --git a/src/agent/tasks.js b/src/agent/tasks.js index 6d968a9..1b9d56e 100644 --- a/src/agent/tasks.js +++ b/src/agent/tasks.js @@ -109,11 +109,11 @@ export class Task { await new Promise((resolve) => setTimeout(resolve, 500)); if (this.data.agent_count > 1) { - var initial_inventory = this.data.initial_inventory[this.agent.count_id.toString()]; + let initial_inventory = this.data.initial_inventory[this.agent.count_id.toString()]; console.log("Initial inventory:", initial_inventory); } else if (this.data) { console.log("Initial inventory:", this.data.initial_inventory); - var initial_inventory = this.data.initial_inventory; + let initial_inventory = this.data.initial_inventory; } if ("initial_inventory" in this.data) { diff --git a/viewer.html b/viewer.html index 737b046..bdf2047 100644 --- a/viewer.html +++ b/viewer.html @@ -26,9 +26,9 @@