mirrors/mindcraft
1
0
Fork 0
mirror of https://github.com/kolbytn/mindcraft.git synced 2025-08-28 01:43:05 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: add vision_model param to profile

Browse source
This commit is contained in:
gmuffiness 2025-02-09 21:57:45 +09:00
parent a22f9d439f
commit 2b5923f98f
3 changed files with 12 additions and 3 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
src/agent/vision_interpreter.js
View file

@ -37,7 +37,7 @@ export class VisionInterpreter {
filename = await camera.capture(); filename = await camera.capture();
} }
if (!this.allow_vision || !this.agent.prompter.chat_model.sendVisionRequest) { if (!this.allow_vision || !this.agent.prompter.vision_model.sendVisionRequest) {
log(this.agent.bot, "Vision is disabled. Using text-based environment description instead."); log(this.agent.bot, "Vision is disabled. Using text-based environment description instead.");
log(this.agent.bot, this._nearbyBlocks()); log(this.agent.bot, this._nearbyBlocks());
} else { } else {
@ -54,7 +54,7 @@ export class VisionInterpreter {
let filename = await camera.capture(); let filename = await camera.capture();
if (!this.allow_vision || !this.agent.prompter.chat_model.sendVisionRequest) { if (!this.allow_vision || !this.agent.prompter.vision_model.sendVisionRequest) {
log(this.agent.bot, "Vision is disabled. Using text-based environment description instead."); log(this.agent.bot, "Vision is disabled. Using text-based environment description instead.");
log(this.agent.bot, this._nearbyBlocks()); log(this.agent.bot, this._nearbyBlocks());
} else { } else {
@ -70,7 +70,7 @@ export class VisionInterpreter {
const bot = this.agent.bot; const bot = this.agent.bot;
const imageBuffer = fs.readFileSync(`${this.fp}/${filename}.jpg`); const imageBuffer = fs.readFileSync(`${this.fp}/${filename}.jpg`);
const messages = this.agent.history.getHistory(); const messages = this.agent.history.getHistory();
res = await this.agent.prompter.chat_model.sendVisionRequest(messages, prompt, imageBuffer); res = await this.agent.prompter.vision_model.sendVisionRequest(messages, prompt, imageBuffer);
log(bot, res); log(bot, res);
} catch (error) { } catch (error) {
log(this.agent.bot, `Error analyzing image: ${error.message}`); log(this.agent.bot, `Error analyzing image: ${error.message}`);

1
src/models/mistral.js
View file

@ -47,6 +47,7 @@ export class Mistral {
]; ];
messages.push(...strictFormat(turns)); messages.push(...strictFormat(turns));
console.log('Awaiting mistral api response...')
const response = await this.#client.chat.complete({ const response = await this.#client.chat.complete({
model, model,
messages, messages,

8
src/models/prompter.js
View file

@ -65,6 +65,14 @@ export class Prompter {
this.code_model = this.chat_model; this.code_model = this.chat_model;
} }
if (this.profile.vision_model) {
let vision_model_profile = this._selectAPI(this.profile.vision_model);
this.vision_model = this._createModel(vision_model_profile);
}
else {
this.vision_model = this.chat_model;
}
let embedding = this.profile.embedding; let embedding = this.profile.embedding;
if (embedding === undefined) { if (embedding === undefined) {
if (chat_model_profile.api !== 'ollama') if (chat_model_profile.api !== 'ollama')