I have been working with Monty for the past month, and I am still unsure of when to use different motor/habitat/policy/dataset system configs or what is required based on my experiment/LM/SM configs. I feel like that could be something to mention in the docs to lower the learning curve or make it more beginner-friendly.
2 Likes
Yes, clarity around configs is one of the significant challenges that we’ll have to figure out. This may or may not help, but one thing I found helpful to me was to write out the entire configuration without any inheritance, in order to see what’s what. You can see this approach in our everything_is_awesome experiment configuration.
everything_is_awesome_eval = dict(
experiment_class=EverythingIsAwesomeExperiment,
experiment_args=dict(
do_train=False,
do_eval=True,
max_eval_steps=500,
max_train_steps=1000,
max_total_steps=4 * 500,
n_eval_epochs=1,
n_train_epochs=1,
min_lms_match=1,
model_name_or_path=model_path_everything_is_awesome,
seed=1337,
show_sensor_output=True, # Use this for online visualization
),
logging_config=dict(
monty_log_level="BASIC",
monty_handlers=[
BasicCSVStatsHandler,
DetailedJSONHandler,
ReproduceEpisodeHandler,
],
wandb_handlers=[],
python_log_level="INFO",
python_log_to_file=True,
python_log_to_stdout=True,
output_dir=os.path.join(monty_models_dir, "everything_is_awesome"),
run_name="",
resume_wandb_run=False,
wandb_id=wandb.util.generate_id(),
wandb_group="debugging",
log_parallel_wandb=False,
),
monty_config=dict(
monty_class=MontyForEvidenceGraphMatching,
monty_args=dict(
num_exploratory_steps=1_000,
min_eval_steps=20,
min_train_steps=10,
max_total_steps=2_500,
),
learning_module_configs=dict(
learning_module_0=dict(
learning_module_class=EvidenceGraphLM,
learning_module_args=dict(
max_match_distance=0.05, # TODO: Will this work for radii units?
tolerances=dict(
patch=dict(
hsv=np.array([0.1, 0.2, 0.2]),
principal_curvatures_log=np.ones(2),
)
),
feature_weights=dict(
patch=dict(
hsv=np.array([1, 0.5, 0.5]),
)
),
x_percent_threshold=20,
max_nneighbors=10,
evidence_update_threshold="80%",
max_graph_size=0.3, # TODO: Will this work for radii units?
num_model_voxels_per_dim=100,
gsg_class=EvidenceGoalStateGenerator,
gsg_args=dict(
goal_tolerances=dict(
location=0.015, # TODO: Will this work for radii units?
),
elapsed_steps_factor=10,
min_post_goal_success_steps=5,
x_percent_scale_factor=0.75,
desired_object_distance=0.03, # TODO: Will this work for radii units? # noqa: E501
),
),
),
),
sensor_module_configs=dict(
sensor_module_0=dict(
sensor_module_class=FeatureChangeSM,
sensor_module_args=dict(
sensor_module_id=SENSOR_ID,
features=[
# morphological featuers (necessary)
"pose_vectors",
"pose_fully_defined",
"on_object",
# non-morphological features (optional)
"object_coverage",
"min_depth",
"mean_depth",
"hsv",
"principal_curvatures",
"principal_curvatures_log",
],
delta_thresholds=dict(
on_object=0,
n_steps=20,
hsv=[0.1, 0.1, 0.1],
pose_vectors=[np.pi / 4, np.pi * 2, np.pi * 2],
principal_curvatures_log=[2, 2],
distance=0.01,
),
surf_agent_sm=False,
save_raw_obs=False,
),
)
),
motor_system_config=dict(
motor_system_class=MotorSystem,
motor_system_args=dict(
policy_class=EverythingIsAwesomePolicy,
policy_args=dict(
action_sampler_class=EverythingIsAwesomeActionSampler,
action_sampler_args={},
agent_id=AGENT_ID,
switch_frequency=1.0,
),
),
),
sm_to_agent_dict=dict(
patch=AGENT_ID,
),
sm_to_lm_matrix=[[0]],
lm_to_lm_matrix=None,
lm_to_lm_vote_matrix=None,
),
dataset_class=EnvironmentDataset,
dataset_args=dict(
env_init_func=EverythingIsAwesomeEnvironment,
env_init_args=dict(
actuator_server_uri=ACTUATOR_SERVER_URI,
depth_server_uri=DEPTH_SERVER_URI,
pitch_diameter_rr=PITCH_DIAMETER_RR,
rgb_server_uri=RGB_SERVER_URI,
),
transform=[
DepthTo3DLocations(
agent_id=AGENT_ID,
void_value=DEPTH_VOID_VALUE_RR,
get_all_points=True,
resolutions=[SENSOR_RESOLUTION],
sensor_ids=[SENSOR_ID],
use_semantic_sensor=False,
world_coord=True,
),
],
rng=None,
),
eval_dataloader_class=EverythingIsAwesomeDataLoader,
eval_dataloader_args=dict(
object_name="tissue_box",
),
)
1 Like
Hi @sebastianruizsebas thanks for the feedback! Adding more documentation around customizing configs and simplifying them in general is something on our roadmap. One other resource to highlight besides what @tslominski recommended is this video https://youtu.be/x0e5SBY2nu8?si=MjSMFAlNl5m_9QQJ where I talk a bit about how our configs are structured. Maybe this is helpful until we improve our config setup and documentation.
Best wishes,
Viviane
1 Like