@InProceedings{pmlr-v139-zha21a,  title = 	 {DouZero: Mastering DouDizhu with Self-Play Deep Reinforcement Learning},  author =       {Zha, Daochen and Xie, Jingru and Ma, Wenye and Zhang, Sheng and Lian, Xiangru and Hu, Xia and Liu, Ji},  booktitle = 	 {Proceedings of the 38th International Conference on Machine Learning},  pages = 	 {12333--12344},  year = 	 {2021},  editor = 	 {Meila, Marina and Zhang, Tong},  volume = 	 {139},  series = 	 {Proceedings of Machine Learning Research},  month = 	 {18--24 Jul},  publisher =    {PMLR},  pdf = 	 {http://proceedings.mlr.press/v139/zha21a/zha21a.pdf},  url = 	 {http://proceedings.mlr.press/v139/zha21a.html},  abstract = 	 {Games are abstractions of the real world, where artificial agents learn to compete and cooperate with other agents. While significant achievements have been made in various perfect- and imperfect-information games, DouDizhu (a.k.a. Fighting the Landlord), a three-player card game, is still unsolved. DouDizhu is a very challenging domain with competition, collaboration, imperfect information, large state space, and particularly a massive set of possible actions where the legal actions vary significantly from turn to turn. Unfortunately, modern reinforcement learning algorithms mainly focus on simple and small action spaces, and not surprisingly, are shown not to make satisfactory progress in DouDizhu. In this work, we propose a conceptually simple yet effective DouDizhu AI system, namely DouZero, which enhances traditional Monte-Carlo methods with deep neural networks, action encoding, and parallel actors. Starting from scratch in a single server with four GPUs, DouZero outperformed all the existing DouDizhu AI programs in days of training and was ranked the first in the Botzone leaderboard among 344 AI agents. Through building DouZero, we show that classic Monte-Carlo methods can be made to deliver strong results in a hard domain with a complex action space. The code and an online demo are released at https://github.com/kwai/DouZero with the hope that this insight could motivate future work.}}

git clone https://github.com/kwai/DouZero.git

cd douzeropip3 install -r requirements.txt

pip3 install douzero

pip3 install douzero -i https://pypi.tuna.tsinghua.edu.cn/simple

pip3 install -e .

python3 train.py

python3 train.py --gpu_devices 0,1,2,3 --num_actor_devices 3 --num_actors 15 --training_device 3

python3 train.py --actor_device_cpu --training_device cpu

python3 train.py --actor_device_cpu

--xpid XPID           实验id（默认值：douzero）--save_interval SAVE_INTERVAL                      保存模型的时间间隔（以分钟为单位）--objective {adp,wp}  使用ADP或者WP作为奖励（默认值：ADP）--actor_device_cpu    用CPU进行模拟--gpu_devices GPU_DEVICES                      用作训练的GPU设备名--num_actor_devices NUM_ACTOR_DEVICES                      被用来进行模拟（如自我对弈）的GPU数量--num_actors NUM_ACTORS                      每个设备的演员进程数--training_device TRAINING_DEVICE                      用来进行模型训练的设备。`cpu`表示用CPU训练--load_model          读取已有的模型--disable_checkpoint  禁用保存检查点--savedir SAVEDIR     实验数据存储跟路径--total_frames TOTAL_FRAMES                      Total environment frames to train for--exp_epsilon EXP_EPSILON                      探索概率--batch_size BATCH_SIZE                      训练批尺寸--unroll_length UNROLL_LENGTH                      展开长度（时间维度）--num_buffers NUM_BUFFERS                      共享内存缓冲区的数量--num_threads NUM_THREADS                      学习者线程数--max_grad_norm MAX_GRAD_NORM                      最大梯度范数--learning_rate LEARNING_RATE                      学习率--alpha ALPHA         RMSProp平滑常数--momentum MOMENTUM   RMSProp momentum--epsilon EPSILON     RMSProp epsilon

python3 generate_eval_data.py

python3 evaluate.py

python3 evaluate.py --landlord baselines/douzero_ADP/landlord.ckpt --landlord_up random --landlord_down random

python3 evaluate.py --landlord rlcard --landlord_up baselines/douzero_ADP/landlord_up.ckpt --landlord_down baselines/douzero_ADP/landlord_down.ckpt

sh get_most_recent.sh douzero_checkpoints/douzero/