The short summary:
# install mpv with "with-shared" option
brew install --with-vapoursynth --with-shared mpv
# save libmpv from Plex
mv "/Applications/Plex Media Player.app/Contents/Frameworks/libmpv.1.dylib" "/Applications/Plex Media Player.app/Contents/Frameworks/libmpv.1.backup"
# replace it with the one from brew
ln -s /usr/local/Cellar/mpv/0.17.0/lib/libmpv.1.dylib "/Applications/Plex Media Player.app/Contents/Frameworks/libmpv.1.dylib"
# create mpv's config file
echo "input-ipc-server=mpvpipe" >> ~/"Library/Application Support/Plex Media Player/mpv.conf"