From 287c69b2f1801a62bf1b76405e6ee18f4f950849 Mon Sep 17 00:00:00 2001 From: silencht Date: Mon, 28 Oct 2024 17:00:25 +0800 Subject: [PATCH] [update] README and image show related. --- .gitignore | 3 +- README.md | 91 ++++++++++++++----------- README_zh-CN.md | 100 ++++++++++++++++------------ teleop/image_server/image_client.py | 3 +- teleop/teleop_hand_and_arm.py | 40 ++++++----- 5 files changed, 133 insertions(+), 104 deletions(-) diff --git a/.gitignore b/.gitignore index 9ad2808..b747247 100644 --- a/.gitignore +++ b/.gitignore @@ -11,4 +11,5 @@ __MACOSX/ *.jpg !video_cover.jpg *.svo -*.png \ No newline at end of file +*.png +teleop/data/ \ No newline at end of file diff --git a/README.md b/README.md index 4574edd..96f0137 100644 --- a/README.md +++ b/README.md @@ -57,30 +57,39 @@ We tested our code on Ubuntu 20.04 and Ubuntu 22.04, other operating systems may For more information, you can refer to [Official Documentation ](https://support.unitree.com/home/zh/Teleoperation) and [OpenTeleVision](https://github.com/OpenTeleVision/TeleVision). - - ## 1.1 🦾 inverse kinematics ```bash -conda create -n tv python=3.8 -conda activate tv +unitree@Host:~$ conda create -n tv python=3.8 +unitree@Host:~$ conda activate tv # If you use `pip install`, Make sure pinocchio version is 3.1.0 -conda install pinocchio -c conda-forge -pip install meshcat -pip install casadi +(tv) unitree@Host:~$ conda install pinocchio -c conda-forge +(tv) unitree@Host:~$ pip install meshcat +(tv) unitree@Host:~$ pip install casadi ``` -## 1.2 🕹️ unitree_dds_wrapper +> p.s. All identifiers in front of the command are meant for prompting: **Which device and directory the command should be executed on**. +> +In the Ubuntu system's `~/.bashrc` file, the default configuration is: `PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ '` +> +> Taking the command `(tv) unitree@Host:~$ pip install meshcat` as an example: +> +> - `(tv)` Indicates the shell is in the conda environment named `tv`. +>- `unitree@Host:~` Shows the user `\u` `unitree` is logged into the device `\h` `Host`, with the current working directory `\w` as `$HOME`. +> - `$` shows the current shell is Bash (for non-root users). +> - `pip install meshcat` is the command `unitree` wants to execute on `Host`. +> +> You can refer to [Harley Hahn's Guide to Unix and Linux](https://www.harley.com/unix-book/book/chapters/04.html#H) and the [Conda User Guide](https://docs.conda.io/projects/conda/en/latest/user-guide/getting-started.html) to learn more. + +## 1.2 🕹️ unitree_sdk2_python ```bash -# Install the Python version of the unitree_dds_wrapper. -git clone https://github.com/unitreerobotics/unitree_dds_wrapper.git -cd unitree_dds_wrapper/python -pip install -e . +# Install unitree_sdk2_python. +(tv) unitree@Host:~$ git clone https://github.com/unitreerobotics/unitree_sdk2_python.git +(tv) unitree@Host:~$ cd unitree_sdk2_python +(tv) unitree@Host:~$ pip install -e . ``` -> p.s. This is a temporary version, and it will be replaced with [unitree_sdk2_python](https://github.com/unitreerobotics/unitree_sdk2_python) in the future. - # 2. 🛠️ TeleVision and Apple Vision Pro configuration @@ -88,11 +97,10 @@ pip install -e . ## 2.1 📥 basic ```bash -cd ~ -git clone https://github.com/unitreerobotics/avp_teleoperate.git -cd ~/avp_teleoperate -pip install -r requirements.txt -cd act/detr && pip install -e . +(tv) unitree@Host:~$ cd ~ +(tv) unitree@Host:~$ git clone https://github.com/unitreerobotics/avp_teleoperate.git +(tv) unitree@Host:~$ cd ~/avp_teleoperate +(tv) unitree@Host:~$ pip install -r requirements.txt ``` ## 2.2 🔌 Local streaming @@ -103,35 +111,35 @@ cd act/detr && pip install -e . 2. check **Host machine** local ip address: ```bash -ifconfig | grep inet +(tv) unitree@Host:~/avp_teleoperate$ ifconfig | grep inet ``` Suppose the local ip address of the **Host machine** is `192.168.123.2` -> p.s. you can use `ifconfig` command to check your **Host machine** ip address. +> p.s. You can use `ifconfig` command to check your **Host machine** ip address. 3. create certificate: ```bash -mkcert -install && mkcert -cert-file cert.pem -key-file key.pem 192.168.123.2 localhost 127.0.0.1 +(tv) unitree@Host:~/avp_teleoperate$ mkcert -install && mkcert -cert-file cert.pem -key-file key.pem 192.168.123.2 localhost 127.0.0.1 ``` place the generated `cert.pem` and `key.pem` files in `teleop` ```bash -cp cert.pem key.pem ~/avp_teleoperate/teleop/ +(tv) unitree@Host:~/avp_teleoperate$ cp cert.pem key.pem ~/avp_teleoperate/teleop/ ``` 4. open firewall on server: ```bash -sudo ufw allow 8012 +(tv) unitree@Host:~/avp_teleoperate$ sudo ufw allow 8012 ``` 5. install ca-certificates on Apple Vision Pro: -``` -mkcert -CAROOT +```bash +(tv) unitree@Host:~/avp_teleoperate$ mkcert -CAROOT ``` Copy the `rootCA.pem` via AirDrop to Apple Vision Pro and install it. @@ -149,14 +157,14 @@ This step is to verify that the environment is installed correctly. Extracting to the current directory, go to the `IsaacGym_Preview_4_Package/isaacgym/python` directory and execute the command: ```bash - pip install -e . + (tv) unitree@Host:~/IsaacGym_Preview_4_Package/isaacgym/python$ pip install -e . ``` 2. After setup up streaming with local following the above instructions, you can try teleoperating two robot hands in Issac Gym: ```bash - cd teleop - python teleop_test_gym.py + (tv) unitree@Host:~/avp_teleoperate$ cd teleop + (tv) unitree@Host:~/avp_teleoperate/teleop$ python teleop_test_gym.py ``` 3. Wear your Apple Vision Pro device. @@ -188,34 +196,36 @@ Copy `image_server.py` in the `avp_teleoperate/teleop/image_server` directory to # p.s.2 The image transfer program is currently configured for binocular rgb cameras. # Now located in Unitree Robot PC2 terminal -python image_server.py +unitree@PC2:~/image_server$ python image_server.py # You can see the terminal output as follows: # Image server has started, waiting for client connections... -# Image Resolution: width is x, height is x +# Image Resolution: width is 640, height is 480 ``` After image service is started, you can use `image_client.py` **in the Host** terminal to test whether the communication is successful: ```bash -python image_client.py +(tv) unitree@Host:~/avp_teleoperate/teleop/image_server$ python image_client.py ``` ## 3.2 ✋ Inspire hands Server (optional) +> Note: If the selected robot configuration does not use the Inspire dexterous hand, please ignore this section. + You can refer to [Dexterous Hand Development](https://support.unitree.com/home/zh/H1_developer/Dexterous_hand) to configure related environments and compile control programs. First, use [this URL](https://oss-global-cdn.unitree.com/static/0a8335f7498548d28412c31ea047d4be.zip) to download the dexterous hand control interface program. Copy it to **PC2** of Unitree robots. On Unitree robot's **PC2**, execute command: ```bash -sudo apt install libboost-all-dev libspdlog-dev +unitree@PC2:~$ sudo apt install libboost-all-dev libspdlog-dev # Build project -cd h1_inspire_service & mkdir build & cd build -cmake .. -DCMAKE_BUILD_TYPE=Release -make +unitree@PC2:~$ cd h1_inspire_service & mkdir build & cd build +unitree@PC2:~/h1_inspire_service/build$ cmake .. -DCMAKE_BUILD_TYPE=Release +unitree@PC2:~/h1_inspire_service/build$ make # Terminal 1. Run h1 inspire hand service -sudo ./inspire_hand -s /dev/ttyUSB0 +unitree@PC2:~/h1_inspire_service/build$ sudo ./inspire_hand -s /dev/ttyUSB0 # Terminal 2. Run example -./h1_hand_example +unitree@PC2:~/h1_inspire_service/build$ ./h1_hand_example ``` If two hands open and close continuously, it indicates success. Once successful, close the `./h1_hand_example` program in Terminal 2. @@ -227,13 +237,16 @@ If two hands open and close continuously, it indicates success. Once successful, > 1. Everyone must keep a safe distance from the robot to prevent any potential danger! > > 2. Please make sure to read the [Official Documentation](https://support.unitree.com/home/zh/Teleoperation) at least once before running this program. +> +> 3. Always make sure that the robot has entered [debug mode (L2+R2)](https://support.unitree.com/home/zh/H1_developer/Remote_control) to stop the motion control program, this will avoid potential command conflict problems. +> It's best to have two operators to run this program, referred to as **Operator A** and **Operator B**. Now, **Operator B** execute the following command on **Host machine** : ```bash -python teleop_hand_and_arm.py +(tv) unitree@Host:~/avp_teleoperate/teleop$ python teleop_hand_and_arm.py --record ``` And then, **Operator A** diff --git a/README_zh-CN.md b/README_zh-CN.md index ed6694e..b996457 100644 --- a/README_zh-CN.md +++ b/README_zh-CN.md @@ -58,25 +58,39 @@ ## 1.1 🦾 逆运动学 ```bash -conda create -n tv python=3.8 -conda activate tv +unitree@Host:~$ conda create -n tv python=3.8 +unitree@Host:~$ conda activate tv # 如果您使用 `pip install`，请确保 pinocchio 版本为 3.1.0 -conda install pinocchio -c conda-forge -pip install meshcat -pip install casadi +(tv) unitree@Host:~$ conda install pinocchio -c conda-forge +(tv) unitree@Host:~$ pip install meshcat +(tv) unitree@Host:~$ pip install casadi ``` -## 1.2 🕹️ unitree_dds_wrapper +> 提醒：命令前面的所有标识符是为了提示：该命令应该在哪个设备和目录下执行。 +> +> p.s. 在 Ubuntu 系统 `~/.bashrc` 文件中，默认配置: `PS1='${debian_chroot:+($debian_chroot)}\u@\h:\w\$ '` + +> - 以`(tv) unitree@Host:~$ pip install meshcat` 命令为例： +> +>- `(tv)` 表示 shell 此时位于 conda 创建的 tv 环境中； +> +>- `unitree@Host:~` 表示用户标识 unitree 在设备 Host 上登录，当前的工作目录为 `$HOME`； +> +>- $ 表示当前 shell 为 Bash； +> +>- pip install meshcat 是用户标识 unitree 要在 设备 Host 上执行的命令。 +> +>您可以参考 [Harley Hahn's Guide to Unix and Linux](https://www.harley.com/unix-book/book/chapters/04.html#H) 和 [Conda User Guide](https://docs.conda.io/projects/conda/en/latest/user-guide/getting-started.html) 来深入了解这些知识。 + +## 1.2 🕹️ unitree_sdk2_python ```bash -# 安装 unitree_dds_wrapper 的 Python 版本 -git clone https://github.com/unitreerobotics/unitree_dds_wrapper.git -cd unitree_dds_wrapper/python -pip install -e . +# 安装 unitree_sdk2_python 库 +(tv) unitree@Host:~$ git clone https://github.com/unitreerobotics/unitree_sdk2_python.git +(tv) unitree@Host:~$ cd unitree_sdk2_python +(tv) unitree@Host:~$ pip install -e . ``` -> 注意：这是一个临时版本，未来将被 [unitree_sdk2_python](https://github.com/unitreerobotics/unitree_sdk2_python) 取代。 - # 2. 🛠️ TeleVision 和 Apple Vision Pro 配置 @@ -84,11 +98,10 @@ pip install -e . ## 2.1 📥 基础 ```bash -cd ~ -git clone https://github.com/unitreerobotics/avp_teleoperate.git -cd ~/avp_teleoperate -pip install -r requirements.txt -cd act/detr && pip install -e . +(tv) unitree@Host:~$ cd ~ +(tv) unitree@Host:~$ git clone https://github.com/unitreerobotics/avp_teleoperate.git +(tv) unitree@Host:~$ cd ~/avp_teleoperate +(tv) unitree@Host:~$ pip install -r requirements.txt ``` ## 2.2 🔌 本地流媒体 @@ -99,35 +112,35 @@ cd act/detr && pip install -e . 2. 检查**主机**本地 IP 地址： ```bash -ifconfig | grep inet +(tv) unitree@Host:~/avp_teleoperate$ ifconfig | grep inet ``` 假设 **主机** 的本地 IP 地址为 `192.168.123.2` -> 注意：您可以使用 `ifconfig` 命令检查您的 **主机** IP 地址。 +> 提醒：您可以使用 `ifconfig` 命令检查您的 **主机** IP 地址。 3. 创建证书： ```bash -mkcert -install && mkcert -cert-file cert.pem -key-file key.pem 192.168.123.2 localhost 127.0.0.1 +(tv) unitree@Host:~/avp_teleoperate$ mkcert -install && mkcert -cert-file cert.pem -key-file key.pem 192.168.123.2 localhost 127.0.0.1 ``` 将生成的 `cert.pem` 和 `key.pem` 文件放在 `teleop` 目录中 ```bash -cp cert.pem key.pem ~/avp_teleoperate/teleop/ +(tv) unitree@Host:~/avp_teleoperate$ cp cert.pem key.pem ~/avp_teleoperate/teleop/ ``` 4. 在服务器上打开防火墙： ```bash -sudo ufw allow 8012 +(tv) unitree@Host:~/avp_teleoperate$ sudo ufw allow 8012 ``` 5. 在 Apple Vision Pro 上安装 CA 证书： ```bash -mkcert -CAROOT +(tv) unitree@Host:~/avp_teleoperate$ mkcert -CAROOT ``` 通过 AirDrop 将 `rootCA.pem` 复制到 Apple Vision Pro 并安装它。 @@ -145,21 +158,21 @@ mkcert -CAROOT 解压到当前目录，进入 `IsaacGym_Preview_4_Package/isaacgym/python` 目录，执行命令： ```bash - pip install -e . + (tv) unitree@Host:~/IsaacGym_Preview_4_Package/isaacgym/python$ pip install -e . ``` 2. 按照上述说明设置本地流媒体后，您可以尝试在 Isaac Gym 中远程操作两个机器人手： ```bash - cd teleop - python teleop_test_gym.py + (tv) unitree@Host:~/avp_teleoperate$ cd teleop + (tv) unitree@Host:~/avp_teleoperate/teleop$ python teleop_test_gym.py ``` 3. 戴上您的 Apple Vision Pro 设备。 4. 在 Apple Vision Pro 上打开 Safari，访问：https://192.168.123.2:8012?ws=wss://192.168.123.2:8012 - > 注意：此 IP 地址应与您的 **主机** IP 地址匹配。 + > 提醒：此 IP 地址应与您的 **主机** IP 地址匹配。 5. 点击 `Enter VR` 并选择 `Allow` 以启动 VR 会话。 @@ -183,51 +196,54 @@ mkcert -CAROOT # 提醒2：目前该图像传输程序是为双目RGB相机配置的。 # 现在位于宇树机器人 PC2 终端 -python image_server.py +unitree@PC2:~/image_server$ python image_server.py # 您可以看到终端输出如下： -# 图像服务器已启动，等待客户端连接... -# 图像分辨率：宽度为 x，高度为 x +# Image server has started, waiting for client connections... +# Image Resolution: width is 640, height is 480 ``` 在图像服务启动后，您可以在 **主机** 终端上使用 `image_client.py` 测试通信是否成功： ```bash -python image_client.py +(tv) unitree@Host:~/avp_teleoperate/teleop/image_server$ python image_client.py ``` ## 3.2 ✋ Inspire 手部服务器（可选） +> 注意：如果选择的机器人配置中没有使用 Inspire 灵巧手，那么请忽略本节内容。 + 您可以参考 [灵巧手开发](https://support.unitree.com/home/zh/H1_developer/Dexterous_hand) 配置相关环境并编译控制程序。首先，使用 [此链接](https://oss-global-cdn.unitree.com/static/0a8335f7498548d28412c31ea047d4be.zip) 下载灵巧手控制接口程序，然后将其复制到宇树机器人的**PC2**。 在宇树机器人的 **PC2** 上，执行命令： ```bash -sudo apt install libboost-all-dev libspdlog-dev +unitree@PC2:~$ sudo apt install libboost-all-dev libspdlog-dev # 构建项目 -cd h1_inspire_service && mkdir build && cd build -cmake .. -DCMAKE_BUILD_TYPE=Release -make +unitree@PC2:~$ cd h1_inspire_service & mkdir build & cd build +unitree@PC2:~/h1_inspire_service/build$ cmake .. -DCMAKE_BUILD_TYPE=Release +unitree@PC2:~/h1_inspire_service/build$ make # 终端 1. 运行 h1 inspire 手部服务 -sudo ./inspire_hand -s /dev/ttyUSB0 +unitree@PC2:~/h1_inspire_service/build$ sudo ./inspire_hand -s /dev/ttyUSB0 # 终端 2. 运行示例 -./h1_hand_example +unitree@PC2:~/h1_inspire_service/build$ ./h1_hand_example ``` -如果两只手连续打开和关闭，则表示成功。一旦成功，关闭终端 2 中的 `./h1_hand_example` 程序。 +如果两只手连续打开和关闭，则表示成功。一旦成功，即可关闭终端 2 中的 `./h1_hand_example` 程序。 ## 3.3 🚀 启动 > ![Warning](https://img.shields.io/badge/Warning-Important-red) > -> 1. 所有人员必须与机器人保持安全距离，以防止任何潜在的危险！ -> 2. 在运行此程序之前，请确保至少阅读一次 [官方文档](https://support.unitree.com/home/zh/Teleoperation)。 +> 1. 所有人员必须与机器人保持安全距离，以防止任何潜在的危险！ +> 2. 在运行此程序之前，请确保至少阅读一次 [官方文档](https://support.unitree.com/home/zh/Teleoperation)。 +> 3. 请务必确保机器人已经进入[调试模式（L2+R2）](https://support.unitree.com/home/zh/H1_developer/Remote_control)，以停止运动控制程序发送指令，这样可以避免潜在的指令冲突问题。 最好有两名操作员来运行此程序，称为 **操作员 A** 和 **操作员 B**。 现在，**操作员 B** 在 **主机** 上执行以下命令： ```bash -python teleop_hand_and_arm.py +(tv) unitree@Host:~/avp_teleoperate/teleop$ python teleop_hand_and_arm.py --record ``` 然后，**操作员 A**： @@ -254,7 +270,7 @@ python teleop_hand_and_arm.py > > 为了避免损坏机器人，最好确保**操作员 A** 将机器人手臂摆放为自然下垂或其他恰当位置后，**操作员B **再按 **q** 退出。 -要退出程序，**操作员 B** 可以在“record image”窗口中按下 **q** 键。 +要退出程序，**操作员 B** 可以在 'record image' 窗口中按下 **q** 键。 diff --git a/teleop/image_server/image_client.py b/teleop/image_server/image_client.py index 8471b3a..f98b3b8 100644 --- a/teleop/image_server/image_client.py +++ b/teleop/image_server/image_client.py @@ -146,7 +146,8 @@ class ImageClient: np.copyto(self._img_array, np.array(current_image)) if self._image_show: - cv2.imshow('Image Client Stream', current_image) + resized_image = cv2.resize(current_image, (current_image.shape[1] // 2, current_image.shape[0] // 2)) + cv2.imshow('Image Client Stream', resized_image) if cv2.waitKey(1) & 0xFF == ord('q'): self.running = False diff --git a/teleop/teleop_hand_and_arm.py b/teleop/teleop_hand_and_arm.py index 37b176c..045115b 100644 --- a/teleop/teleop_hand_and_arm.py +++ b/teleop/teleop_hand_and_arm.py @@ -87,6 +87,24 @@ if __name__ == '__main__': # print(f"ik:\t{round(time_ik_end - time_ik_start, 6)}") arm_ctrl.ctrl_dual_arm(sol_q, sol_tauff) + resized_image = cv2.resize(img_array, (img_shape[1] // 2, img_shape[0] // 2)) + cv2.imshow("record image", resized_image) + key = cv2.waitKey(1) & 0xFF + if key == ord('q'): + running = False + elif key == ord('s') and args.record: + press_key_s_count += 1 + if press_key_s_count % 2 == 1: + print("==> Start recording...") + recording = True + recorder.create_episode() + print("==> Create episode ok.") + else: + print("==> End recording...") + recording = False + recorder.save_episode() + print("==> Save episode ok.") + # record data if args.record: with dual_hand_data_lock: @@ -101,23 +119,6 @@ if __name__ == '__main__': left_arm_action = sol_q[:7] right_arm_action = sol_q[-7:] - cv2.imshow("record image", current_image) - key = cv2.waitKey(1) & 0xFF - if key == ord('q'): - running = False - elif key == ord('s'): - press_key_s_count += 1 - if press_key_s_count % 2 == 1: - print("==> Start recording...") - recording = True - recorder.create_episode() - print("==> Create episode ok.") - else: - print("==> End recording...") - recording = False - recorder.save_episode() - print("==> Save episode ok.") - if recording: colors = {} depths = {} @@ -181,12 +182,9 @@ if __name__ == '__main__': # print(f"main process sleep: {sleep_time}") except KeyboardInterrupt: - img_shm.unlink() - img_shm.close() print("KeyboardInterrupt, exiting program...") - exit(0) finally: - img_shm.unlink() img_shm.close() + img_shm.unlink() print("Finally, exiting program...") exit(0) \ No newline at end of file