Edit

As the intention was to make a line selection algorithm, the previous solution did not fully meet, making the new solution simpler.

In the new solution, Adapter has SparseArray (could be HashMap , but SparseArray is more recommended on the Android platform).

For each position, SparseArray or save 1 for position if it is checked or 0 otherwise.

As requested, the second click unchecks the item, such as a checkbox.

public class TestBaseAdapter extends SimpleAdapter {

    // Armazena as linhas que possuem cliques, para marcação
    SparseArray<Integer> mCliques = new SparseArray<Integer>();

    // Usado para adicionar um valor a uma View, no metodo setTag
    int mResId;

    public TestBaseAdapter(Context context, List<? extends Map<String, ?>> data, int resource, String[] from, int[] to) {
        super(context, data, resource, from, to);
        mResId = resource;
    }

    @Override
    public View getView(int position, View convertView, ViewGroup parent) {
        View view = super.getView(position, convertView, parent);

        updateBackground(view, position);

        // Guardo na View, sua posicao
        // Facilita ao fazer a limpeza manual do background caso seja necessario
        view.setTag(mResId, position);

        return view;
    }

    /***
     * Atualiza o background da View conforme a regra de cliques
     * @param view
     * @param position
     */
    void updateBackground(View view, int position) {
        // Se houve nao tem clique, entao remove o background
        if(mCliques.get(position, 0) == 0) {
            view.setBackgroundResource(0);
        } else { // Adiciona o background em caso positivo
            view.setBackgroundResource(R.color.primary);
        }
    }

    /***
     * Atualiza a ultima posicao clicada e o numero de cliques.
     * E atualiza o Background da view conforme o resultado
     * @param adapterView
     * @param view
     * @param position
     */
    public void updateClick(AdapterView adapterView, View view, int position) {

        // Se a posicao foi clicada e ja estava marcada, remove a marcação
        if(mCliques.get(position, -1) != -1) {
            mCliques.remove(position);
        } else { // Adiciona a marcação
            mCliques.put(position, 1);
        }

        // Podemos usar o "notifyDataSetChanged", com isso todas as Views
        // que estao visiveis serao reconstruidas
        // notifyDataSetChanged();

        // Ou podemos atualizar manualmente as visiveis, as demais
        // serao construídas pelo Adapter
        updateVisibleViews(adapterView);
    }

    void updateVisibleViews(AdapterView adapterView) {
        for(int i = 0, childCount = adapterView.getChildCount(); i < childCount; ++i) {
            View view = adapterView.getChildAt(i);
            int position = (Integer) view.getTag(mResId);

            updateBackground(view, position);
        }
    }

    /***
     * Retorna a lista dos indices dos itens selecionados
     * @return
     */
    public ArrayList<Integer> getSelectedItems() {
        ArrayList<Integer> selecao = new ArrayList<Integer>(mCliques.size());

        for(int i = 0, size = mCliques.size(); i < size; ++i) {
            selecao.add(mCliques.keyAt(i));
        }

        return selecao;
    }
}

To accomplish this task, you need to create a subclass of SimpleAdapter , because we need to incorporate the "coloring" rule into Adapter , making it simpler and giving responsibility for the correct element.

I've done an example of how it can be done, my solution is just a base, just adapt to your specific case.

Since I do not have the same data as you, I had to improvise:

This is the construction of Adapter and data of ListView in my Activity :

final ArrayList<HashMap<String, String>> songsListData = new ArrayList<HashMap<String, String>>();

// Construindo dados fake, apenas para ter um ListView cheio
for (int i = 0; i < 100; i++) {
    // creating new HashMap
    HashMap<String, String> song = new HashMap<String, String>();

    song.put("NUMERO", "" + i);

    songsListData.add(song);
}

ListView lv = (ListView) view.findViewById(R.id.fpa_list);

// Crio o meu Adapter customizado, que herda de SimpleAdapter
mAdapter = new TestBaseAdapter(getActivity(), songsListData, R.layout.playlist_item, new String[]{"NUMERO"}, new int[] {android.R.id.text1});

lv.setAdapter(mAdapter);

// Seto um OnItemClickListener apenas para delegar ao Adapter a atualizacao
// do ultimo item clicado e do numero de cliques
lv.setOnItemClickListener(new AdapterView.OnItemClickListener() {
    @Override
    public void onItemClick(AdapterView<?> parent, View view, int position, long id) {
        mAdapter.updateClick(parent, view, position);
    }
});

Now comes the SimpleAdapter that I applied to apply the click rule:

public class TestBaseAdapter extends SimpleAdapter {

    // Armazena a quantidade de clicks em uma determinada linha e sua posicao
    int mUltimaPosicaoClicada = -1, mNumeroCliques = 0;

    // Usado para adicionar um valor a uma View, no metodo setTag
    int mResId;

    public TestBaseAdapter(Context context, List<? extends Map<String, ?>> data, int resource, String[] from, int[] to) {
        super(context, data, resource, from, to);
        mResId = resource;
    }

    @Override
    public View getView(int position, View convertView, ViewGroup parent) {
        View view = super.getView(position, convertView, parent);

        updateBackground(view, position);

        // Guardo na View, sua posicao
        // Facilita ao fazer a limpeza manual do background caso seja necessario
        view.setTag(mResId, position);

        return view;
    }

    /***
     * Atualiza o background da View conforme a regra de cliques
     * @param view
     * @param position
     */
    void updateBackground(View view, int position) {
        if(position == mUltimaPosicaoClicada) {
            switch (mNumeroCliques) {
                case 0:
                    view.setBackgroundResource(0);
                case 1:
                    view.setBackgroundResource(R.color.primary);
                    break;
                case 2:
                default:
                    view.setBackgroundResource(R.color.black_87p);
                    break;
            }
        } else {
            view.setBackgroundResource(0);
        }
    }

    /***
     * Atualiza a ultima posicao clicada e o numero de cliques.
     * E atualiza o Background da view conforme o resultado
     * @param adapterView
     * @param view
     * @param position
     */
    public void updateClick(AdapterView adapterView, View view, int position) {
        // Logica para guardar o numero de cliques e o ultimo item clicado
        if(mUltimaPosicaoClicada == position) {
            mNumeroCliques++;
        } else {
            mUltimaPosicaoClicada = position;
            mNumeroCliques = 1;
        }

        // Podemos usar o "notifyDataSetChanged", com isso todas as Views
        // que estao visiveis serao reconstruidas
        // notifyDataSetChanged();

        // Ou podemos atualizar manualmente as visiveis, as demais
        // serao construídas pelo Adapter
        updateVisibleViews(adapterView);
    }

    void updateVisibleViews(AdapterView adapterView) {
        for(int i = 0, childCount = adapterView.getChildCount(); i < childCount; ++i) {
            View view = adapterView.getChildAt(i);
            int position = (Integer) view.getTag(mResId);

            updateBackground(view, position);
        }
    }
}

I'd like to leave a note:

In the updateClick of TestAdapter method, I left the items update to be done in two ways: