HTML - Use HttpURLConnection to log in to site

Most of the time the simple way you used does not work, because:

• A navigation flow from one page to another may be required to create session variables, which are checked by the site.
• Currently sites use a lot of asynchronous data generation and you want to see the resulting source code after these asynchronous calls.
• Sometimes websites use HTTPS.

There is a very easy way to do what you want. Just use the Apache HttpAsyncClient library. You must use the HttpAsyncClient 4.1-beta1 .

Here is the code to copy and paste: (there are 3 classes)

1 - TesteTravian.java:

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.net.URLEncoder;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.util.concurrent.ExecutionException;

public class TesteTravian {

    public static void loginTravian(){
        MyNavigator navigator = null;

        try{
            navigator = new MyNavigator();

            String resp = navigator.navigateTo("http://ts5.travian.com.br");

            String charset = "UTF-8";
            String param_name = "vinmor";
            String param_password = "321321";

            String strLogin = "<input type=\"hidden\" name=\"login\" value=\""; 
            int index1 = resp.indexOf(strLogin)+strLogin.length();
            int index2 = resp.indexOf("\"",index1);
            String param_login = resp.substring(index1,index2); 

            String urlParameters = String.format("name=%s&password=%s&login=%s", 
                    URLEncoder.encode(param_name, charset), 
                    URLEncoder.encode(param_password, charset),
                    URLEncoder.encode(param_login, charset));

            //Ao enviar o post, ele retorna uma mensagem de erro, porém o login foi efetuado com sucesso.
            //A partir daí os novos GETS irão funcionar
            navigator.navigateToPost("http://ts5.travian.com.br/dorf1.php", urlParameters);

            //Vai para a página: Recursos
            resp = navigator.navigateTo("http://ts5.travian.com.br/dorf1.php");                 
            System.out.println(resp);           

            //Como fazer para ir para outras páginas...         
            //Vai para a página: Centro da Aldeia
            //resp = navigator.navigateTo("http://ts5.travian.com.br/dorf2.php");
        }
        catch(InterruptedException | ExecutionException | KeyManagementException | NoSuchAlgorithmException | KeyStoreException | UnsupportedEncodingException e){
            e.printStackTrace();
        }
        finally{
            if(navigator!=null){ try{navigator.close();}catch(IOException e){e.printStackTrace();}}
        }
    }

    public static void main(final String[] args){
        loginTravian();
        System.exit(0);
    }
}

2 - MyNavigator.java

import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLException;
import javax.net.ssl.SSLSession;
import javax.net.ssl.SSLSocket;
import org.apache.http.conn.ssl.SSLContexts;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.conn.ssl.X509HostnameVerifier;
import org.apache.http.entity.ContentType;
import org.apache.http.impl.nio.client.CloseableHttpAsyncClient;
import org.apache.http.impl.nio.client.HttpAsyncClients;
import org.apache.http.nio.client.methods.HttpAsyncMethods;
import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy;

public class MyNavigator{
    private CloseableHttpAsyncClient httpclient;

    public MyNavigator() throws KeyManagementException, NoSuchAlgorithmException, KeyStoreException{
        SSLContext sslcontext = SSLContexts.custom().useTLS().loadTrustMaterial(null, new TrustStrategy(){
            @Override
            public boolean isTrusted(X509Certificate[] chain, String authType) throws CertificateException{
                return true;
            }
        }).build();

        SSLIOSessionStrategy sslSessionStrategy = new SSLIOSessionStrategy(sslcontext, new AllowAll());
        this.httpclient = HttpAsyncClients.custom()
                .setUserAgent("Mozilla/5.0 (Windows NT 6.1; WOW64; Trident/7.0; rv:11.0) like Gecko")
                .setSSLStrategy(sslSessionStrategy)
                .build();

        httpclient.start();
    }

    public String navigateTo(String endereco) throws InterruptedException, ExecutionException{
        MyResponseConsumer myResponseConsumer = new MyResponseConsumer();

        Future<Boolean> future = httpclient.execute(HttpAsyncMethods.createGet(endereco), myResponseConsumer, null);
        Boolean result;

        result = future.get();
        if(result != null && result.booleanValue()){
            return myResponseConsumer.getResp();
        }
        return null;
    }

    public String navigateToPost(String endereco, String postsValues) throws InterruptedException, ExecutionException,
    UnsupportedEncodingException{
        MyResponseConsumer myResponseConsumer = new MyResponseConsumer();

        Future<Boolean> future = httpclient.execute(HttpAsyncMethods.createPost(endereco, postsValues, ContentType.APPLICATION_FORM_URLENCODED), myResponseConsumer, null);
        Boolean result;

        result = future.get();
        if(result != null && result.booleanValue()){
            return myResponseConsumer.getResp();
        }
        return null;
    }

    public void close() throws IOException{
        if(httpclient != null)
            httpclient.close();
    }

    private static class AllowAll implements X509HostnameVerifier{
        @Override
        public void verify(String s, SSLSocket sslSocket) throws IOException{/* Nothing */
        }

        @Override
        public void verify(String s, X509Certificate x509Certificate) throws SSLException{/* Nothing */
        }

        @Override
        public void verify(String s, String[] strings, String[] strings2) throws SSLException{/* Nothing */
        }

        @Override
        public boolean verify(String s, SSLSession sslSession){
            return true;
        }
    }
}

3 - MyResponseConsumer.java

import java.io.IOException;
import java.nio.CharBuffer;
import org.apache.http.HttpResponse;
import org.apache.http.nio.IOControl;
import org.apache.http.nio.client.methods.AsyncCharConsumer;
import org.apache.http.protocol.HttpContext;

public class MyResponseConsumer extends AsyncCharConsumer<Boolean>{

    private String resp;    

    public MyResponseConsumer(){
        this.resp = "";
    }

    @Override
    protected void onResponseReceived(final HttpResponse response) {
        //nothing
    }

    @Override
    protected void onCharReceived(final CharBuffer buf, final IOControl ioctrl) throws IOException {
        while (buf.hasRemaining()) {
            resp += buf.get();
        }
    }

    @Override
    protected void releaseResources() {
        //nothing
    }

    @Override
    protected Boolean buildResult(final HttpContext context) {
        return Boolean.TRUE;
    }

    public String getResp(){
        return this.resp;
    }    
}