I’ve found this gist with commands sequence but it didn’t work for me, I’ve added two small fixes (whitespaces) to one command, so now the command sequence looks like this:

$ echo "deb http://apt.postgresql.org/pub/repos/apt/ precise-pgdg main" | sudo $ tee /etc/apt/sources.list.d/pgdg.list
$ wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo $ apt-key add -
$ sudo apt-get update
$ sudo apt-get install postgresql-9.3 postgresql-server-dev-9.3 $ postgresql-contrib-9.3 -y
# ...
$ sudo su - postgres -c "psql template1 -p 5433 -c 'CREATE EXTENSION IF NOT $ EXISTS hstore;'"
$ sudo su - postgres -c "psql template1 -p 5433 -c 'CREATE EXTENSION IF NOT $ EXISTS \"uuid-ossp\";'"
$ sudo su - postgres -c "service postgresql stop"
$ sudo su - postgres -c '/usr/lib/postgresql/9.3/bin/pg_upgrade -b /usr/lib/$ postgresql/9.1/bin -B /usr/lib/postgresql/9.3/bin -d /var/lib/postgresql/9.1/$ main/ -D /var/lib/postgresql/9.3/main/ -O " -c config_file=/etc/postgresql/9.3/$ main/postgresql.conf" -o " -c config_file=/etc/postgresql/9.1/main/$ postgresql.conf"'
# ...
$ sudo apt-get remove postgresql-9.1 -y
$ sudo sed -i "s:5433:5432:g" /etc/postgresql/9.3/main/postgresql.conf
$ sudo service postgresql restart