mirrors/mindcraft
1
0
Fork 0
mirror of https://github.com/kolbytn/mindcraft.git synced 2025-04-29 19:44:53 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

wrote validation functions for blueprint and added commands for the agent to use

Browse source
This commit is contained in:
Isadora White 2024-12-26 18:12:11 -06:00
parent 095cdc2d6d
commit acdc73ad47
8 changed files with 567 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
andy.json
View file

@ -1,6 +1,30 @@
{
"name": "andy",
"model": "gpt-4o"
"model": "gpt-4o",
"conversation_examples": [
[
{"role": "assistant", "content": " I've got stone. What about you jack?"},
{"role": "assistant", "content": "I have a door and planks. Great we have everything we need from the blueprint. Let's start building!"},
{"role": "assistant", "content": "Great! I'll start by placing the stone blocks for level 0. !newAction(\"Place stone blocks for the base level.\")"},
{"role": "assistant", "content": "Since the blueprint for level 1 only has stone, I'll start placing those. !newAction(\"Place stone blocks for level 1.\")"},
{"role": "assistant", "content": "jill: I'll place the planks for level 2. !newAction(\"Place planks for level 2.\")"}
],
[
{"role": "assistant", "content": "I've got some oak planks for level 2. Let's start building the base with stone. Since I don't have stone, could you help out? "},
{"role": "assistant", "content": "For sure! I will build levels 0 and 1 with stone and let you know when I am done. !newAction(\"Place stone blocks for levels 0 and 1 \")"},
{"role": "assistant", "content": "I've placed the stone blocks for levels 0 and 1. Your turn!"},
{"role": "assistant", "content": "I'll start placing the planks for level 2. !newAction(\"Place plank blocks for level 2\")"}
],
[
{"role": "assistant", "content": "Looks like level 0 requires stone, since I only have oak planks and no stone, I will let you build that. Let me know when you are done!"},
{"role": "assistant", "content": "Ok, placing for level 0 at the specified coordinates now !newAction(\"Place stone blocks for level 0\")"},
{"role": "assistant", "content": "Looks like level 1 requires stone, since I only have oak planks and no stone, I will let you build that. Let me know when you are done!"},
{"role": "assistant", "content": "I've placed the planks for level 1. Your turn!"},
{"role": "assistant", "content": "Great work! I'll start placing the planks for level 2. !newAction(\"Place plank blocks for level 2\")"}
]
]
}

65
example_tasks.json
View file

@ -36,6 +36,7 @@
},
"multiagent_techtree_1_stone_pickaxe": {
"conversation": "Let's collaborate to build a stone pickaxe",
"goal": "Build a stone pickaxe",
"agent_count": 2,
"initial_inventory": {
"0": {
@ -53,6 +54,7 @@
"construction_house": {
"type": "construction",
"goal": "Make a house with the blueprint below",
"agent_count": 1,
"blueprint": {
"materials": {
"plank": {
@ -104,5 +106,68 @@
"oak_planks": 40,
"oak_door": 1
}
},
"multiagent_construction_house": {
"type": "construction",
"goal": "Make a house with the blueprint above, tell the other agents what materials you have and share them to finish the blueprint. You and the other agent have all the necessary materials for the blueprint at this point. ",
"conversation": "Let's share materials and make a house with this blueprint. You and the other agent have all materials necessary at this point:",
"agent_count": 2,
"blueprint": {
"materials": {
"plank": {
"id": "oak_plank",
"number": 20
},
"door": {
"id": "oak_door",
"number": 1
}
},
"levels": [
{
"level": 0,
"coordinates": [142, -60, -179],
"placement":
[
["stone", "stone", "door", "stone", "stone"],
["stone", "air", "air", "air", "stone"],
["stone", "air", "air", "air", "stone"],
["stone", "stone", "stone", "stone", "stone"]
]
},
{
"level": 1,
"coordinates": [142, -59, -179],
"placement":
[
["stone", "stone", "air", "stone", "stone"],
["stone", "air", "air", "air", "stone"],
["stone", "air", "air", "air", "stone"],
["stone", "stone", "stone", "stone", "stone"]
]
},
{
"level": 2,
"coordinates": [142, -58, -179],
"placement":
[
["plank", "plank", "plank", "plank", "plank"],
["plank", "plank", "plank", "plank", "plank"],
["plank", "plank", "plank", "plank", "plank"],
["plank", "plank", "plank", "plank", "plank"]
]
}
]
},
"initial_inventory": {
"0": {
"oak_planks": 20
},
"1": {
"oak_door": 1,
"stone": 26
}
}
}
}

29
jill.json Normal file
View file

@ -0,0 +1,29 @@
{
"name": "jill",
"model": "gpt-4o",
"conversation_examples": [
[
{"role": "assistant", "content": " I've got stone. What about you jack?"},
{"role": "assistant", "content": "I have a door and planks. Great we have everything we need from the blueprint. Let's start building!"},
{"role": "assistant", "content": "Great! I'll start by placing the stone blocks for level 0. !newAction(\"Place stone blocks for the base level.\")"},
{"role": "assistant", "content": "Since the blueprint for level 1 only has stone, I'll start placing those. !newAction(\"Place stone blocks for level 1.\")"},
{"role": "assistant", "content": "jill: I'll place the planks for level 2. !newAction(\"Place planks for level 2.\")"}
],
[
{"role": "assistant", "content": "I've got some oak planks for level 2. Let's start building the base with stone. Since I don't have stone, could you help out? "},
{"role": "assistant", "content": "For sure! I will build levels 0 and 1 with stone and let you know when I am done. !newAction(\"Place stone blocks for levels 0 and 1 \")"},
{"role": "assistant", "content": "I've placed the stone blocks for levels 0 and 1. Your turn!"},
{"role": "assistant", "content": "I'll start placing the planks for level 2. !newAction(\"Place plank blocks for level 2\")"}
],
[
{"role": "assistant", "content": "Looks like level 0 requires stone, since I only have oak planks and no stone, I will let you build that. Let me know when you are done!"},
{"role": "assistant", "content": "Ok, placing for level 0 at the specified coordinates now !newAction(\"Place stone blocks for level 0\")"},
{"role": "assistant", "content": "Looks like level 1 requires stone, since I only have oak planks and no stone, I will let you build that. Let me know when you are done!"},
{"role": "assistant", "content": "I've placed the planks for level 1. Your turn!"},
{"role": "assistant", "content": "Great work! I'll start placing the planks for level 2. !newAction(\"Place plank blocks for level 2\")"}
]
]
}

1
settings.js
View file

@ -12,6 +12,7 @@ export default
"profiles": [
"./andy.json",
"./jill.json",
// "./profiles/gpt.json",
// "./profiles/claude.json",
// "./profiles/gemini.json",

29
src/agent/commands/actions.js
View file

@ -1,6 +1,7 @@
import * as skills from '../library/skills.js';
import settings from '../../../settings.js';
import convoManager from '../conversation.js';
import { checkLevelBlueprint, checkBlueprint } from '../tasks.js';
function runAsAction (actionFn, resume = false, timeout = -1) {
let actionLabel = null; // Will be set on first use
@ -407,6 +408,34 @@ export const actionsList = [
convoManager.endConversation(player_name);
return `Converstaion with ${player_name} ended.`;
}
},
{
name: '!checkLevelComplete',
description: 'Check if the level is complete and what blocks still need to be placed for the blueprint',
perform: runAsAction(async (agent, levelNum) => {
return await checkLevelBlueprint(agent, levelNum);
})
},
{
name: '!checkBlueprint',
description: 'Check what blocks still need to be placed for the blueprint',
perform: runAsAction(async (agent) => {
return await checkBlueprint(agent);
})
},
{
name: '!getBlueprint',
description: 'Get the blueprint for the building',
perform: runAsAction(async (agent) => {
return await agent.task.blueprint.explain();
})
},
{
name: '!getBlueprintLevel',
description: 'Get the blueprint for the building',
perform: runAsAction(async (agent, levelNum) => {
return await agent.task.blueprint.explainLevel(levelNum);
})
}
// { // commented for now, causes confusion with goal command
// name: '!npcGoal',

158
src/agent/tasks.js
View file

@ -2,8 +2,9 @@ import { readFileSync } from 'fs';
import { executeCommand } from './commands/index.js';
import { getPosition } from './library/world.js'
import settings from '../../settings.js';
import { check } from 'yargs';
export class TaskValidator {
export class CraftTaskValidator {
constructor(data, agent) {
this.target = data.target;
this.number_of_target = data.number_of_target;
@ -35,13 +36,48 @@ export class TaskValidator {
}
}
export class ConstructionTaskValidator {
constructor(data, agent) {
this.blueprint = new Blueprint(data.blueprint);
this.agent = agent;
}
validate() {
try {
//todo: somehow make this more of a percentage or something
let valid = false;
this.blueprint.checkBluepint(this.agent.bot).then((result) => {
if (result.mismatches.length === 0) {
valid = true;
console.log('Task is complete');
}
});
return valid;
} catch (error) {
console.error('Error validating task:', error);
return false;
}
}
}
export async function checkLevelBlueprint(agent, levelNum) {
const blueprint = agent.task.blueprint.data;
const bot = agent.bot;
const levelData = blueprint.levels[levelNum];
return Blueprint.checkLevelBlueprint(bot, levelData);
}
export async function checkBlueprint(agent) {
const blueprint = agent.task.blueprint.data;
const bot = agent.bot;
return Blueprint.checkBlueprint(bot, blueprint);
}
export class Blueprint {
constructor(blueprint) {
this.blueprint = blueprint;
this.data = blueprint;
}
explain() {
var explanation = "";
for (let item of this.blueprint.levels) {
for (let item of this.data.levels) {
var coordinates = item.coordinates;
explanation += `Level ${item.level}: `;
explanation += `Start at coordinates X: ${coordinates[0]}, Y: ${coordinates[1]}, Z: ${coordinates[2]}`;
@ -64,6 +100,97 @@ export class Blueprint {
placement_string += "]";
return placement_string;
}
explainLevel(levelNum) {
const levelData = this.data.levels[levelNum];
var explanation = `Level ${levelData.level} `;
explanation += `at coordinates X: ${levelData.coordinates[0]}, Y: ${levelData.coordinates[1]}, Z: ${levelData.coordinates[2]}`;
let placement_string = this._getPlacementString(levelData.placement);
explanation += `\n${placement_string}\n`;
}
async explainBlueprintDifference(bot, blueprint) {
var explanation = "";
for (let levelData of blueprint.levels) {
let level_explanation = await this.explainLevelDifference(bot, levelData);
explanation += level_explanation + "\n";
}
return explanation;
}
async explainLevelDifference(bot, levelData) {
const results = await this.checkLevelBlueprint(bot, levelData);
const mismatches = results.mismatches;
if (mismatches.length === 0) {
return `Level ${levelData.level} is correct`;
}
var explanation = `Level ${levelData.level} `;
explanation += `at coordinates X: ${levelData.coordinates[0]}, Y: ${levelData.coordinates[1]}, Z: ${levelData.coordinates[2]}`;
explanation += " has the following mismatches:\n";
for (let item of mismatches) {
explanation += `At coordinates X: ${item.coordinates[0]}, Y: ${item.coordinates[1]}, Z: ${item.coordinates[2]}`;
explanation += `Expected ${item.expected}, but found ${item.actual}\n`;
}
return explanation;
}
async checkBluepint(bot) {
const levels = this.data.levels;
const mismatches = [];
const matches = [];
for (let i = 0; i < levels.length; i++) {
const levelData = levels[i];
const result = await checkLevelBlueprint(bot, levelData);
mismatches.push(...result.mismatches);
matches.push(...result.matches);
}
return {
"mismatches": mismatches,
"matches": matches
};
}
async checkLevelBlueprint(bot, levelData) {
const startCoords = levelData.coordinates;
const placement = levelData.placement;
const mismatches = [];
const matches = [];
for (let zOffset = 0; zOffset < placement.length; zOffset++) {
const row = placement[zOffset];
for (let xOffset = 0; xOffset < row.length; xOffset++) {
const blockName = row[xOffset];
const x = startCoords[0] + xOffset;
const y = startCoords[1];
const z = startCoords[2] + zOffset;
try {
const blockAtLocation = await bot.blockAt(new Vec3(x, y, z));
if (!blockAtLocation || blockAtLocation.name !== blockName) {
mismatches.push({
level: levelData.level,
coordinates: [x, y, z],
expected: blockName,
actual: blockAtLocation ? bot.registry.blocks[blockAtLocation.type].name : 'air' // Assuming air if no block
});
} else {
matches.push({
level: levelData.level,
coordinates: [x, y, z],
expected: blockName,
actual: blockAtLocation ? bot.registry.blocks[blockAtLocation.type].name : 'air' // Assuming air if no block
});
}
} catch (err) {
console.error(`Error getting block at (${x}, ${y}, ${z}):`, err);
return false; // Stop checking if there's an issue getting blocks
}
}
}
return {
"mismatches": mismatches,
"matches": matches
};
}
}
export class Task {
@ -77,27 +204,27 @@ export class Task {
if (task_path && task_id) {
this.data = this.loadTask(task_path, task_id);
this.task_type = this.data.type;
console.log('Task type:', this.task_type);
if (this.task_type === 'construction' && this.data.blueprint) {
//add the blueprint to the goal if it is a construction task
//todo: fix this for multi-agent scenarios with partial blueprints
this.blueprint = new Blueprint(this.data.blueprint);
console.log('Blueprint:', this.blueprint.explain());
this.goal = this.data.goal + ' \n' + this.blueprint.explain();
console.log('Goal:', this.goal);
this.goal = this.data.goal + ' \n' + this.blueprint.explain() + " \n" + "make sure to place the lower levels of the blueprint first";
this.conversation = this.data.conversation + ' \n' + this.blueprint.explain();
} else {
this.goal = this.data.goal;
console.log('Goal:', this.goal);
this.conversation = this.data.conversation;
}
this.taskTimeout = this.data.timeout || 300;
this.taskStartTime = Date.now();
this.validator = new TaskValidator(this.data, this.agent);
if (this.task_type === 'construction') {
this.validator = new ConstructionTaskValidator(this.data, this.agent);
} else if (this.task_type === 'techtree') {
this.validator = new CraftTaskValidator(this.data, this.agent);
}
this.blocked_actions = this.data.blocked_actions || [];
if (this.goal)
this.blocked_actions.push('!endGoal');
if (this.data.conversation)
if (this.conversation)
this.blocked_actions.push('!endConversation');
console.log('Task loaded:', this.data);
}
}
@ -106,6 +233,7 @@ export class Task {
const tasksFile = readFileSync(task_path, 'utf8');
const tasks = JSON.parse(tasksFile);
const task = tasks[task_id];
console.log('Loaded task:', task);
if (!task) {
throw new Error(`Task ${task_id} not found`);
}
@ -231,9 +359,9 @@ export class Task {
await executeCommand(this.agent, `!goal("${this.goal}")`);
}
if (this.data.conversation && this.agent.count_id === 0) {
if (this.conversation && this.agent.count_id === 0) {
let other_name = available_agents.filter(n => n !== name)[0];
await executeCommand(this.agent, `!startConversation("${other_name}", "${this.data.conversation}")`);
await executeCommand(this.agent, `!startConversation("${other_name}", "${this.conversation}")`);
}
}
}

128
test/test_check_blocks.js Normal file
View file

@ -0,0 +1,128 @@
import mineflayer from 'mineflayer';
import { Vec3 } from 'vec3';
const bot = mineflayer.createBot({
host: 'localhost', // Replace with your server IP or hostname
port: 55916, // Replace with your server port
username: 'andy', // Replace with your bot's username
// password: 'your_bot_password' // Only if the server has online-mode=true
});
bot.on('spawn', async () => {
console.log("Bot spawned. Starting blueprint check...");
await new Promise((resolve) => setTimeout(resolve, 10000));
const blockAtLocation = await bot.blockAt(new Vec3(142, -60, -179));
console.log(blockAtLocation);
const results = await checkLevelBlueprint(bot, blueprintData.levels[0]);
console.log(results)
const matchesBlueprint = await checkBluepint(bot, blueprintData);
console.log(`Blueprint check result: \n`);
console.log(matchesBlueprint.mismatches)
bot.quit();
});
async function checkBluepint(bot, blueprintData) {
const materials = blueprintData.materials;
const levels = blueprintData.levels;
const mismatches = [];
const matches = [];
for (let i = 0; i < levels.length; i++) {
const levelData = levels[i];
const result = await checkLevelBlueprint(bot, levelData);
mismatches.push(...result.mismatches);
matches.push(...result.matches);
}
return {
"mismatches": mismatches,
"matches": matches
};
}
async function checkLevelBlueprint(bot, levelData) {
const startCoords = levelData.coordinates;
const placement = levelData.placement;
const mismatches = [];
const matches = [];
for (let zOffset = 0; zOffset < placement.length; zOffset++) {
const row = placement[zOffset];
for (let xOffset = 0; xOffset < row.length; xOffset++) {
const blockName = row[xOffset];
const x = startCoords[0] + xOffset;
const y = startCoords[1];
const z = startCoords[2] + zOffset;
try {
const blockAtLocation = await bot.blockAt(new Vec3(x, y, z));
if (!blockAtLocation || blockAtLocation.name !== blockName) {
mismatches.push({
level: levelData.level,
coordinates: [x, y, z],
expected: blockName,
actual: blockAtLocation ? bot.registry.blocks[blockAtLocation.type].name : 'air' // Assuming air if no block
});
} else {
matches.push({
level: levelData.level,
coordinates: [x, y, z],
expected: blockName,
actual: blockAtLocation ? bot.registry.blocks[blockAtLocation.type].name : 'air' // Assuming air if no block
});
}
} catch (err) {
console.error(`Error getting block at (${x}, ${y}, ${z}):`, err);
return false; // Stop checking if there's an issue getting blocks
}
}
}
return {
"mismatches": mismatches,
"matches": matches
};
}
const blueprintData = {
"materials": {
"oak_planks": 20,
"oak_door": 1,
"stone": 26,
},
"levels": [
{
"level": 0,
"coordinates": [142, -60, -179],
"placement":
[
["stone", "stone", "oak_door", "stone", "stone"],
["stone", "air", "air", "air", "stone"],
["stone", "air", "air", "air", "stone"],
["stone", "stone", "stone", "stone", "stone"]
]
},
{
"level": 1,
"coordinates": [142, -59, -179],
"placement":
[
["stone", "stone", "oak_door", "stone", "stone"],
["stone", "air", "air", "air", "stone"],
["stone", "air", "air", "air", "stone"],
["stone", "stone", "stone", "stone", "stone"]
]
},
{
"level": 2,
"coordinates": [142, -58, -179],
"placement":
[
["oak_planks", "oak_planks", "oak_planks", "oak_planks", "oak_planks"],
["oak_planks", "oak_planks", "oak_planks", "oak_planks", "oak_planks"],
["oak_planks", "oak_planks", "oak_planks", "oak_planks", "oak_planks"],
["oak_planks", "oak_planks", "oak_planks", "oak_planks", "oak_planks"]
]
}
]
};

147
test/test_check_blueprint.js Normal file
View file

@ -0,0 +1,147 @@
import mineflayer from 'mineflayer';
import { Vec3 } from 'vec3';
async function checkBuildingBlueprint(bot, blueprint) {
/**
* Checks if a Minecraft building matches a given blueprint using Mineflayer.
*
* @param {mineflayer.Bot} bot - The Mineflayer bot instance.
* @param {object} blueprint - The blueprint object.
* @returns {Promise<boolean>} - True if the building matches, false otherwise.
*/
const mismatches = [];
for (const levelData of blueprint.levels) {
const levelNum = levelData.level;
const startCoords = levelData.coordinates;
const placement = levelData.placement;
for (let zOffset = 0; zOffset < placement.length; zOffset++) {
const row = placement[zOffset];
for (let xOffset = 0; xOffset < row.length; xOffset++) {
const blockName = row[xOffset];
const expectedBlockId = getBlockIdFromName(bot, blockName);
if (expectedBlockId === undefined) {
console.warn(`Warning: Unknown block name '${blockName}' in blueprint. Skipping.`);
continue;
}
const x = startCoords[0] + xOffset;
const y = startCoords[1] + levelNum;
const z = startCoords[2] + zOffset;
try {
const blockAtLocation = await bot.blockAt(new Vec3(x, y, z));
if (!blockAtLocation || blockAtLocation.type !== expectedBlockId) {
mismatches.push({
level: levelNum,
coordinates: [x, y, z],
expected: blockName,
actual: blockAtLocation ? bot.registry.blocks[blockAtLocation.type].name : 'air' // Assuming air if no block
});
}
} catch (err) {
console.error(`Error getting block at (${x}, ${y}, ${z}):`, err);
return false; // Stop checking if there's an issue getting blocks
}
}
}
}
if (mismatches.length > 0) {
console.log("Building does not match the blueprint. Found the following mismatches:");
mismatches.forEach(mismatch => {
console.log(` Level ${mismatch.level}, Coordinates ${mismatch.coordinates.join(', ')}: Expected '${mismatch.expected}', Found '${mismatch.actual}'`);
});
return false;
} else {
console.log("Building matches the blueprint!");
return true;
}
}
function getBlockIdFromName(bot, blockName) {
/**
* Gets the numerical block ID from a string block name using the bot's registry.
* @param {mineflayer.Bot} bot - The Mineflayer bot instance.
* @param {string} blockName - The name of the block (case-insensitive).
* @returns {number|undefined} - The block ID, or undefined if not found.
*/
const blockInfo = bot.registry.blocksByName[blockName.toLowerCase().replace(/ /g, '_')];
return blockInfo ? blockInfo.id : undefined;
}
// Example usage (replace with your bot login and server details)
const blueprintData = {
"materials": {
"plank": {
"id": "oak_plank",
"number": 20
},
"door": {
"id": "oak_door",
"number": 1
}
},
"levels": [
{
"level": 0,
"coordinates": [142, -60, -179],
"placement":
[
["stone", "stone", "oak_door", "stone", "stone"],
["stone", "air", "air", "air", "stone"],
["stone", "air", "air", "air", "stone"],
["stone", "stone", "stone", "stone", "stone"]
]
},
{
"level": 1,
"coordinates": [142, -59, -179],
"placement":
[
["stone", "stone", "air", "stone", "stone"],
["stone", "air", "air", "air", "stone"],
["stone", "air", "air", "air", "stone"],
["stone", "stone", "stone", "stone", "stone"]
]
},
{
"level": 2,
"coordinates": [142, -58, -179],
"placement":
[
["oak_plank", "oak_plank", "oak_plank", "oak_plank", "oak_plank"],
["oak_plank", "oak_plank", "oak_plank", "oak_plank", "oak_plank"],
["oak_plank", "oak_plank", "oak_plank", "oak_plank", "oak_plank"],
["oak_plank", "oak_plank", "oak_plank", "oak_plank", "oak_plank"]
]
}
]
};
const bot = mineflayer.createBot({
host: 'localhost', // Replace with your server IP or hostname
port: 55916, // Replace with your server port
username: 'andy', // Replace with your bot's username
// password: 'your_bot_password' // Only if the server has online-mode=true
});
bot.on('spawn', async () => {
console.log("Bot spawned. Starting blueprint check...");
const matchesBlueprint = await checkBuildingBlueprint(bot, blueprintData);
console.log(`Blueprint check result: ${matchesBlueprint}`);
bot.quit(); // Disconnect the bot after checking
});
bot.on('kicked', (reason, loggedIn) => {
console.log(`Bot kicked: ${reason}, loggedIn: ${loggedIn}`);
});
bot.on('error', err => {
console.log(`Bot error: ${err}`);
});