making eval script more robust to server randomly crashing

evaluation_script.py

@@ -8,6 +8,8 @@ import re
 import sys
 import os
 import time
+import filecmp
+import json
 
 BLOCKED_ACTIONS_COOKING = [
     '!activate', '!attackPlayer', '!checkBlueprint', '!checkBlueprintLevel',
@@ -225,6 +227,8 @@ def launch_server_experiment(task_path,
 
     subprocess.run(['tmux', 'new-session', '-d', '-s', session_name], check=True) 
 
+    
+
     # set environment variables
     set_environment_variable_tmux_session(session_name, "MINECRAFT_PORT", server_port)
     set_environment_variable_tmux_session(session_name, "MINDSERVER_PORT", mindserver_port)
@@ -233,13 +237,14 @@ def launch_server_experiment(task_path,
         set_environment_variable_tmux_session(session_name, "INSECURE_CODING", "true")
 
     # you need to add the bots to the world first before you can add them as op
-    cmd = f"node main.js --task_path example_tasks.json --task_id debug_{num_agents}_agent_timeout"
+    # cmd = f"node main.js --task_path example_tasks.json --task_id debug_{num_agents}_agent_timeout"
 
-    subprocess.run(["tmux", "send-keys", "-t", session_name, cmd, "C-m"])
+    # subprocess.run(["tmux", "send-keys", "-t", session_name, cmd, "C-m"])
 
-    time.sleep(40)
+    # time.sleep(40)
 
-    subprocess.run(["tmux", "send-keys", "-t", "server_" + session_name, f"/op {agent_names[0]}", "C-m"])
+    # subprocess.run(["tmux", "send-keys", "-t", "server_" + session_name, f"/op @a", "C-m"])
+    make_ops(agent_names, session_name)
 
     # add the bots as op
     # op_script_content = "sleep 5\n\op @p" * 20
@@ -252,6 +257,11 @@ def launch_server_experiment(task_path,
     elif task_type == "construction":
         set_environment_variable_tmux_session(session_name, "BLOCKED_ACTIONS", BLOCKED_ACTIONS_CONSTRUCTION)
     
+    split_task_path = task_path.split("/")
+    if len(split_task_path) > 1:
+        task_path_name = split_task_path[-2]
+    else:
+        task_path_name = "tasks"
 
     script_content = ""
     for task_id in task_ids:
@@ -274,8 +284,7 @@ def launch_server_experiment(task_path,
                 script_content += f"{cp_cmd}\n"
                 script_content += "sleep 1\n"
                 if s3:
-                    s3_cmd = f"aws s3 cp {agent_file_path} s3://{bucket_name}/{task_type}/{model}/{exp_name}/{task_id}/{agent}_{_}.json"
+                    s3_cmd = f"aws s3 cp {agent_file_path} s3://{bucket_name}/{task_type}/{model}/{task_path_name}/{exp_name}/{task_id}/{agent}_{_}.json"
-                    s3_upload_experiment = f"aws s3 cp {agent_file_path} s3://{bucket_name}/{task_type}/{model}/{exp_name}/{task_id}/{agent}_{_}.json"
                     script_content += f"echo 'Uploading {agent_file_path} to S3'\n"
                     script_content += f"echo '{s3_cmd}'\n"
                     script_content += f"{s3_cmd}\n"
@@ -283,12 +292,42 @@ def launch_server_experiment(task_path,
         script_content += f"sleep 10\n"
         if s3:
             for agent in agent_names:
-                script_content += f"aws s3 cp bots/{agent} s3://{bucket_name}/{task_type}/{model}/{exp_name}/bots/{agent} --recursive\n"
+                script_content += f"aws s3 cp bots/{agent} s3://{bucket_name}/{task_type}/{model}/{task_path_name}/{exp_name}/bots/{agent} --recursive\n"
 
     # Create a temporary shell script file
     script_file = f"./tmp/experiment_script_{session_name}.sh"
     make_script_file_and_run(script_content, session_name, script_file)
 
+def make_ops(agent_names, session_name):
+    """Make the agents operators in the Minecraft world."""
+    print('Making agents operators...')
+
+    cmd = f"node main.js --task_path example_tasks.json --task_id debug_{len(agent_names)}_agent_timeout"
+
+    subprocess.run(["tmux", "send-keys", "-t", session_name, cmd, "C-m"])
+
+    time.sleep(30)
+
+    subprocess.run(["tmux", "send-keys", "-t", "server_" + session_name, f"/op @a", "C-m"])
+
+    agents_op = check_agent_ops(agent_names, ops_file=f"./server_data_{session_name}/ops.json")
+    if agents_op:
+        print("Agents are operators! You are good to go :D")
+    else: 
+        print("Agents are not operators! Something went wrong :(")
+        make_ops(agent_names, session_name)
+
+def check_agent_ops(agent_names, ops_file="ops.json"):
+    with open(ops_file, "r") as f:
+        ops_data = json.load(f)
+    
+    ops_names = [op["name"] for op in ops_data]
+    
+    for agent in agent_names:
+        if agent not in ops_names:
+            return False 
+    return True
+
 def make_script_file_and_run(script_content, session_name, file_name):
     script_dir = os.path.dirname(file_name)
     os.makedirs(script_dir, exist_ok=True)
@@ -372,6 +411,23 @@ def copy_server_files(source_path, dest_path):
         print(f"Server files copied to {dest_path}")
     except Exception as e:
         print(f"Error copying server files: {e}")
+    time.sleep(10)
+
+    same_files = check_same_files(source_path, dest_path)
+    if not same_files:
+        copy_server_files(source_path, dest_path)
+        print("The destination path does not contain all the same files as the source path.")
+    else:
+        print("The destination path contains all the same files as the source path.")
+
+def check_same_files(d1, d2):
+
+    items1 = set(os.listdir(d1))
+    items2 = set(os.listdir(d2))
+
+    if items1 != items2:
+        return False
+    return True
 
 def delete_server_files(dest_path):
     """Delete server files from the specified location."""