To start your X server, simply add [[ $(fgconsole 2>/dev/null) == 1 ]] && exec startx -- vt1 to your ~/.bash_profile or ~/.zshenv if you use ZSH.
[[ $(fgconsole 2>/dev/null) == 1 ]] && exec startx -- vt1
~/.bash_profile
~/.zshenv