/***************************************************************************
 *   Copyright (C) 2007 by Honorius Galmeanu                               *
 *   honorius_galmeanu@yahoo.com                                           *
 *                                                                         *
 *   This program is free software; you can redistribute it and/or modify  *
 *   it under the terms of the GNU General Public License as published by  *
 *   the Free Software Foundation; either version 2 of the License, or     *
 *   (at your option) any later version.                                   *
 *                                                                         *
 *   This program is distributed in the hope that it will be useful,       *
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of        *
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the         *
 *   GNU General Public License for more details.                          *
 *                                                                         *
 *   You should have received a copy of the GNU General Public License     *
 *   along with this program; if not, write to the                         *
 *   Free Software Foundation, Inc.,                                       *
 *   59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.             *
 *                                                                         *
 *   Copyright: See COPYING file that comes with this distribution         *
 *                                                                         *
 ***************************************************************************/

#include <stdio.h>
#include <stdlib.h>
#include <time.h>
#include <ctype.h>
#include <string.h>
#include <math.h>
#include <assert.h>
#include "idcommon.h"
#include "idsvmr.h"

#define abs(x) (x < 0 ? -x : x)
#define sgn(x) (x < 0 ? -1 : 1)


static int g_dbgLevel = 3;


int main (int argc, char * argv[])
{
	int ret;
	x_svm_r svmr;
	
	svmr.parse_args (argc, argv);
	ret = svmr.read_problem ();
	if (ret) exit (-1);

	svmr.create_kernel ();
	svmr.train ();

	return 0;
}


///////////////////////////////////////////////////////////////////////////////
//
// x_svm_r class
//
///////////////////////////////////////////////////////////////////////////////

void x_svm_r::usage (void)
{
	printf ( 
	"Usage: idsvmr [options] train_file \n"
	"options:\n"
	"-ct kernel : kernel type (default 1)\n"
	"	0 - linear u'*v\n"
	"	1 - RBF: exp(-sigma*|u-v|^2)\n"
	"-cs value : sigma value in kernel function (default 1 / max features)\n"
	"-cc C : value for C in SVM (default 1, auto-adjusts to greater if necessary)\n"
	"-ce e : value for epsilon (default 0.1)\n"
	"-p : precalculate kernel (to save time)\n"
	"-r name-number : initial restore from state rstate-name-number.idsvm\n"
	"-rc name-number : initial restore from Cstate c-rstate-name-number.idsvm\n"
	"-w name-number : saves intermediary and final states to rstate-name-number.idsvm\n"
	"-dc c : decrease parameter cost C to reach c (for restored sessions)\n"
	"-d n : debug level (4..0, 3 - default, 0 - silent)\n"
	"-v : view info of the current context and exit (for restored sessions)\n"
	"-e test_file : go into evaluation mode with test_file (restoration needed)\n"
	);

	exit (1);
}

int x_svm_r::parse_args (int argc, char * argv[])
{
	int i;
	
	if (argc < 2)
	{
		TRACE ("too few parameters");
		usage ();
	}
	
	for (i = 1 ; i < argc ; i ++)
	{
		if (argv[i][0] != '-')
			break;
		
		switch (argv[i][1])
		{
			case 'c':
			{
				switch (argv[i][2])
				{
					case 'c':
						param.C = atof (argv[++ i]);
						break;
					case 's':
						param.sigma = atof (argv[++ i]);
						break;
					case 't':
						param.kernel_type = atoi (argv[++ i]);
						break;
					case 'e':
						eps = atof (argv[++ i]);
						break;
				}
				break;
			}
			case 'r':
				if (argv[i][2] == 'c')
					param.info |= PARAM_RESTORE_C;
				param.restoreFrom = strdup (argv[++ i]);
				break;
			case 'w':
			{
				param.saveTo = strdup (argv[++ i]);
				unsigned int i = 0;
				for (i = 0 ; i < strlen (param.saveTo) ; i ++)
					if (param.saveTo[i] == '-')
						break;
				i ++;
				if (i < strlen (param.saveTo))
					param.saveEvery = atoi (param.saveTo + i);
				break;
			}
			case 'd':
				if (argv[i][2] == 'c')
				{
					param.info |= PARAM_DECREASE_C;
					param.newC = atof (argv[++ i]);
				}
				else
					g_dbgLevel = atoi (argv[++ i]);
				break;
			case 'v':
				param.info |= PARAM_VIEW_STATE;
				break;
			case 'p' :
				param.info |= PARAM_PRECALCULATE;
				break;
			case 'e':
				param.test_file = strdup (argv[++ i]);
				break;
			case 'b':
				// [debug] undocumented
				param.info |= PARAM_BUG_FIND;
				break;
			default:
				TRACE ("invalid option");
				usage ();
		}
	}
	
	if (i >= argc)
		usage ();
	
	// copy input file name
	param.input_file = strdup (argv[i]);

	return 0;
}


int x_svm_r::read_problem (void)
{
	int features, max_index, i, j, eof;

	FILE * fp = fopen (param.input_file, "r");
	
	if (fp == NULL)
	{
		TRACE ("cannot open file %s", param.input_file);
		return -1;
	}

	probl.n = 0;
	features = 0;
	eof = 0;
	while (!eof)
	{
		int c = fgetc (fp);
		switch (c)
		{
			// count also the -1 feature 
			case '\n':
				probl.n ++;
			case ':':
				features ++;
				break;
			case EOF:
				eof = 1;
				break;
		}
	}
	rewind (fp);

	// allocate labels, pointers to patterns and all patterns
	probl.y = ALLOC (float, probl.n);
	probl.x = ALLOC (struct x_svm_node *, probl.n);
	probl.x_all = ALLOC (struct x_svm_node, features);

	max_index = 0;
	j = 0;
	for (i = 0 ; i < probl.n ; i ++)
	{
		float label;
		probl.x[i] = &probl.x_all[j];
		fscanf (fp, "%f", &label);
		probl.y[i] = label;

		while (1)
		{
			int c = fgetc (fp);
			while (isspace (c) && c != '\n')
				c = fgetc (fp);
			if (c == '\n' || c == EOF)
				break;

			// we found something else than whitespace or '\n'
			ungetc (c, fp);
			if (fscanf (fp, "%d:%f", &(probl.x_all[j].idx), &(probl.x_all[j].val)) < 2)
			{
				TRACE ("invalid format at line %d in file %s", i+1, param.input_file);
				return -1;
			}
			j ++;
		}

		if (j >=1 && probl.x_all[j-1].idx > max_index)
			max_index = probl.x_all[j-1].idx;
		probl.x_all[j].idx = -1;
		probl.x_all[j].val = 0;
		j ++;
	}

	fclose (fp);

	// adjust sigma
	if (param.sigma == 0)
		param.sigma = 1.0 / max_index;
	INFO (1, "===[ max_features: %d sigma: %g eps: %g ]==============================\n", max_index, param.sigma, eps);

	return 0;
}


int x_svm_r::read_test (void)
{
	int features, max_index, i, j, eof;

	FILE * fp = fopen (param.test_file, "r");
	
	if (fp == NULL)
	{
		TRACE ("cannot open file %s", param.test_file);
		return -1;
	}

	test.n = 0;
	features = 0;
	eof = 0;
	while (!eof)
	{
		int c = fgetc (fp);
		switch (c)
		{
			// count also the -1 feature 
			case '\n':
				test.n ++;
			case ':':
				features ++;
				break;
			case EOF:
				eof = 1;
				break;
		}
	}
	rewind (fp);

	// allocate labels, pointers to patterns and all patterns
	test.y = ALLOC (float, test.n);
	test.x = ALLOC (struct x_svm_node *, test.n); 
	test.x_all = ALLOC (struct x_svm_node, features);

	max_index = 0;
	j = 0;
	for (i = 0 ; i < test.n ; i ++)
	{
		float label;
		test.x[i] = &test.x_all[j];
		fscanf (fp, "%f", &label);
		test.y[i] = label;

		while (1)
		{
			int c = fgetc (fp);
			while (isspace (c) && c != '\n')
				c = fgetc (fp);
			if (c == '\n' || c == EOF)
				break;

			// we found something else than whitespace or '\n'
			ungetc (c, fp);
			if (fscanf (fp, "%d:%f", &(test.x_all[j].idx), &(test.x_all[j].val)) < 2)
			{
				TRACE ("invalid format at line %d in file %s", i+1, param.input_file);
				return -1;
			}
			j ++;
		}	

		if (j >=1 && test.x_all[j-1].idx > max_index)
			max_index = test.x_all[j-1].idx;
		test.x_all[j].idx = -1;
		test.x_all[j].val = 0;
		j ++;
	}

	fclose (fp);

	return 0;
}


void x_svm_r::print_problem (void)
{
	int i, j, max_features;

	INFO (1, "nr. patterns: %d\n", probl.n);
	max_features = 0;
	for (i = 0 ; i < probl.n ; i ++)
	{
		INFO (1, "%g ", probl.y[i]);
		for (j = 0 ; probl.x[i][j].idx != -1 ; j ++)
			INFO (1, "%d:%g ", probl.x[i][j].idx, probl.x[i][j].val);
		INFO (1, "\n");
		if (j > 0 && probl.x[i][j - 1].idx > max_features)
			max_features = probl.x[i][j - 1].idx;
	}
	INFO (1, "max pattern features: %d\n", max_features); 
}


void x_svm_r::print_test (void)
{
	int i, j, max_features;

	INFO (1, "nr. test patterns: %d\n", test.n);
	max_features = 0;
	for (i = 0 ; i < test.n ; i ++)
	{
		INFO (1, "%g ", test.y[i]);
		for (j = 0 ; test.x[i][j].idx != -1 ; j ++)
			INFO (1, "%d:%g ", test.x[i][j].idx, test.x[i][j].val);
		INFO (1, "\n");
		if (j > 0 && test.x[i][j - 1].idx > max_features)
			max_features = test.x[i][j - 1].idx;
	}
	INFO (1, "max test pattern features: %d\n", max_features); 
}


int x_svm_r::create_kernel (void)
{
	pkern = new x_svm_kernel (probl.n, probl.x, param);
	if (pkern == NULL)
	{
		TRACE ("cannot create kernel");
		exit (1);
	}

	return 0;
}


void x_svm_r::print_patterns (x_vector<int>& perm)
{
	int i, j, max_features;

	printf ("nr. patterns: %d\n", probl.n);
	max_features = 0;
	for (i = 0 ; i < probl.n ; i ++)
	{
		int k = perm[i];
		printf ("%g ", probl.y[k]);
		for (j = 0 ; probl.x[k][j].idx != -1 ; j ++)
			printf ("%d:%g ", probl.x[k][j].idx, probl.x[k][j].val);
		printf ("\n");
		if (j > 0 && probl.x[k][j - 1].idx > max_features)
			max_features = probl.x[k][j - 1].idx;
	}
	printf ("max pattern features: %d\n", max_features); 

}


void x_svm_r::precalculate (void)
{
	// precalculates q_ij
	long len = (probl.n * (probl.n + 1)) / 2;
	INFO (1, "===[ allocating %ld kB for kernel ... ]========================\n", (len * sizeof(double)) / 1024);
	precalc = new double [len];
	if (!precalc)
	{
		TRACE ("cannot allocate space for kernel");
		return;
	}

	getq = new double * [probl.n];
	if (!getq)
	{
		TRACE ("cannot allocate space for pointers to double");
		delete precalc;
		return;
	}
	INFO (1, "===[ allocation succeeded; calculating kernel ... ]==============================\n");

	// now make accelerators
	// first starts from the beginning, with elements from 0 .. n-1
	getq[0] = precalc;
	len = probl.n;
	for (int i = 1 ; i < probl.n ; i ++)
	{
		getq[i] = precalc + len;
		len += probl.n - i;
	}
	if (len != (probl.n * (probl.n + 1)) / 2)
	{
		TRACE ("something's wrong");
		exit (1);
	}
	
	fprintf (stderr, "%3d%% complete\r", 0);
	for (int i = 0 ; i < probl.n ; i ++)
	{
		for (int j = i ; j < probl.n ; j ++)
			getq[i][j - i] = (this->*get_q) (i, j);
		fprintf (stderr, "%3d\r", (i*100) / probl.n);
	}
	fprintf (stderr, "100\n");
	get_q = &x_svm_r::get_q_precalc;

	INFO (1, "===[ precalculation succeeded ]==============================\n");
}


double x_svm_r::calc_h (int i, /* yet unlearned vector */ x_elem * uv)
{
	double sum = 0;
	int j;

	// contribution of margin support vectors
	for (j = 0 ; j < s_vect.len () ; j ++)
	{
		x_elem& l = s_vect[j];
		sum += (this->*get_q) (i, l.idx) * l.lambda;
	}

	// contribution of bounded support vectors
	for (j = 0 ; j < r_vect.len () ; j ++)
	{
		x_elem& l = r_vect[j];
		if (l.type != x_elem::ZERO)
			sum += (this->*get_q) (i, l.idx) * l.lambda;
	}

	// contribution of (yet) unlearned vector
	if (uv)
		sum += (this->*get_q) (i, uv->idx) * uv->lambda;

	sum += w0;
	sum -= probl.y[i];
	return sum;
}


void x_svm_r::create_pq (/* support vector */ int s)
{
	// P and Q are supposed to be just created (uninitialized) when entering this method
	// P = [0 y_s ; y_s Q_ss]
	// Q = [-Q_ss/y_s^2 1/y_s ; 1/y_s 0]

	int n = p.len ();
	if (n != 0)
	{
		TRACE ("P and Q are created starting from scratch");
		exit (1);
	}

	p.expand (2);
	p[0][0] = 0;
	p[0][1] = 1;
	p[1][0] = 1;
	p[1][1] = (this->*get_q) (s, s);

	q.expand (2);
	q[0][0] = - (this->*get_q)(s, s);
	q[0][1] = 1;
	q[1][0] = 1;
	q[1][1] = 0;
}


void x_svm_r::expand_pq (/* support vector */ int k)
{
	//check_pq_consistency ();

	// existing support vectors
	int n = p.len () - 1;

	// initialize column and line from P
	//p.print ();
	p.expand ();
	p[0][n + 1] = 1;
	p[n + 1][0] = 1;
	for (int i = 0 ; i < n ; i ++)
	{
		int s = s_vect[i].idx;
		double val = (this->*get_q) (k, s);
		p[i + 1][n + 1] = val;
		p[n + 1][i + 1] = val;
	}
	p[n + 1][n + 1] = (this->*get_q) (k, k);
	//p.print ();

	// calculate eta_k
	x_line<double> eta_k;
	eta_k.expand (n + 1);
	eta_k[0] = 1;
	for (int i = 0 ; i < n ; i ++)
	{
		int s = s_vect[i].idx;
		eta_k[i + 1] = (this->*get_q) (k, s);
	}
	//eta_k.print ("eta_k");

	// calculate beta_k
	x_line<double> beta_k;
	beta_k.expand (n + 1);
	x_product<double>::mul_ml (q, eta_k, beta_k);
	for (int i = 0 ; i < beta_k.len () ; i ++)
		beta_k[i] *= -1.00;
	//beta_k.print ("beta_k");

	// alpha
	double alpha = 0;
	x_product<double>::dot_ll (eta_k, beta_k, alpha);
	alpha += (this->*get_q) (k, k);
	//INFO (2, "alpha: %g\n", alpha);

	// extend beta_k
	beta_k.expand (1);
	beta_k[beta_k.len () - 1] = 1;
	//beta_k.print ("beta_k ext");

	// calculate extended Q; be sure to border with zero old Q matrix
	//q.print ();
	q.expand ();
	n = q.len () - 1;
	for (int i = 0 ; i < q.len () ; i ++)
		q[n][i] = q[i][n] = 0;
	for (int i = 0 ; i < q.len () ; i ++)
		for (int j = 0 ; j < q.len () ; j ++)
			q[i][j] += (beta_k[i] * beta_k[j]) / alpha;
	//q.print ();

	//check_pq_consistency ();
}


void x_svm_r::contract_pq (/* support vector */ int k)
{
	// remove line & column from P
	p.remove (k + 1);

	// recalculate shrinked Q
	q.remove (k + 1);
	int n = q.len ();
	for (int i = 0 ; i < n ; i ++)
		for (int j = 0 ; j < n ; j ++)
			q[i][j] -= q[i][n] * q[n][j] / q[n][n];
	for (int i = 0 ; i < n ; i ++)
		q[i][n] = q[n][i] = 0;

	//check_pq_consistency ();
}


void x_svm_r::regenerate_pq (void)
{
	INFO (1, "===[ regenerating P and Q ... ]====================================\n");
	p.ensure (0);
	q.ensure (0);

	int idx = s_vect[0].idx;
	create_pq (idx);

	for (int i = 1 ; i < s_vect.len () ; i ++)
	{
		idx = s_vect[i].idx;
		expand_pq (idx);
	}
	INFO (1, "===[ P and Q regenerated. ]====================================\n");
}


void x_svm_r::calculate_eta (/* in */ int k, /* out */ x_line<double>& eta_k)
{
	eta_k.ensure (s_vect.len () + 1);

	eta_k[0] = 1;
	for (int i = 0 ; i < s_vect.len () ; i ++)
	{
		int s = s_vect[i].idx;
		eta_k[i + 1] = (this->*get_q) (k, s);
	}
}


void x_svm_r::calculate_beta (/* in */ x_line<double>& eta_k, /* out */ x_line<double>& beta_k)
{
	beta_k.ensure (eta_k.len ());

	x_product<double>::mul_ml (q, eta_k, beta_k);
	for (int i = 0 ; i < beta_k.len () ; i ++)
		beta_k[i] *= -1.00;
}


void x_svm_r::calculate_gamma (/* in */ int k, /* in */ x_line<double>& beta_k, /* out */ x_line<double>& gamma_k)
{
	int n = s_vect.len ();
	int m = r_vect.len ();
	gamma_k.ensure (m + 1);

	gamma_k[0] = beta_k[0];
	for (int j = 1 ; j < n + 1; j ++)
	{
		int s = s_vect[j - 1].idx;
		gamma_k[0] += (this->*get_q) (k, s) * beta_k[j];
	}

	gamma_k[0] += (this->*get_q) (k, k);
	for (int i = 1 ; i < m + 1 ; i ++)
	{
		int r = r_vect[i - 1].idx;
		gamma_k[i] = beta_k[0];
		for (int j = 1 ; j < n + 1 ; j ++)
		{
			int s = s_vect[j - 1].idx;
			gamma_k[i] += (this->*get_q) (r, s) * beta_k[j];
		}
		gamma_k[i] += (this->*get_q) (r, k);
	}
}


void x_svm_r::limit_first_unbounded (	/* in */ bool incremental, x_elem& uv, x_line<double>& beta_k,
					/* out */ int& sm, int& sp, double& delta_lm, double& delta_lp,
					/* inout */ x_line<int>& dup_m, x_line<int>& dup_p, bool rollback )
{
	// first case: determine which vectors from S are first to reaching limits
	// returns support vector index and associated delta_lambda_k value
	sm = sp = -1;
	delta_lm = delta_lp = 0;
	for (int i = 1 ; i < beta_k.len () ; i ++)
	{
		int iminus = i - 1;
		x_elem& s = s_vect[iminus];
		double h = calc_h (s.idx, &uv);
		double sh = h < 0 ? -1 : 1;
		double sb = beta_k[i] < 0 ? -1 : 1;
	
		if (incremental && sb != sh || !incremental && sb == sh)
		{
			if (	!rollback && (s.lambda == -param.C || s.lambda == param.C) ||
				rollback && s.lambda == 0 )
				continue;

			double delta = (- sh * param.C - s.lambda) / beta_k[i];
			// do not consider a SV with lambda = C, because it was just added from E
// 			if (param.C == s_vect[iminus].lambda)
// 			{
// 				TRACE ("---[skip] vector %d to avoid looping\n", s_vect[iminus].idx);
// 				continue;
// 			}

			INFO (4, "    %d[%g] sp: lambda: %g beta_k: %g delta: %g h: %g\n",
				s.idx, probl.y[s.idx], s.lambda, beta_k[i], delta, h);
			if (sp < 0) {
				sp = iminus;
				delta_lp = delta;
			} else if (IS_ZERO (delta - delta_lp)) {
				// duplicate delta found (seldom)
				if (dup_p.len () == 0) {
					dup_p.add (sp);
					dup_p.add (iminus);
				} else if (dup_p[0] == sp) {
					dup_p.add (iminus);
				} else {
					while (dup_p.len () > 0)
						dup_p.remove (dup_p.len () - 1);
					dup_p.add (sp);
					dup_p.add (iminus);
				}
			} else if (incremental && delta < delta_lp || !incremental && delta > delta_lp) {
				sp = iminus;
				delta_lp = delta;
			}
		} else {
			if (	!rollback && s.lambda == 0 ||
				rollback && (s.lambda == -param.C || s.lambda == param.C) )
				continue;

			double delta = - s.lambda / beta_k[i];

			// do not consider a SV with lambda = 0, because it was just added from O
// 			if (0 == s_vect[iminus].lambda)
// 			{
// 				TRACE ("---[skip] vector %d to avoid looping\n", s_vect[iminus].idx);
// 				continue;
// 			}

			INFO (4, "    %d[%g] sm: lambda: %g beta_k: %g delta: %g h: %g\n",
				s.idx, probl.y[s.idx], s.lambda, beta_k[i], delta, h);
			if (sm < 0)
			{
				// no vector yet
				sm = iminus;
				delta_lm = delta;
			} else if (IS_ZERO (delta - delta_lm)) {
				// duplicate delta found (seldom)
				if (dup_m.len () == 0) {
					dup_m.add (sm);
					dup_m.add (iminus);
				} else if (dup_m[0] == sm) {
					dup_m.add (iminus);
				} else {
					while (dup_m.len () > 0)
						dup_m.remove (dup_m.len () - 1);
					dup_m.add (sm);
					dup_m.add (iminus);
				}
			} else if (incremental && delta < delta_lm || !incremental && delta > delta_lm) {
				sm = iminus;
				delta_lm = delta;
			}
		}
	}
}


void x_svm_r::limit_second_rest ( /* in */ bool incremental,
				/* in */ x_elem& uv, x_line<double>& gamma_k,
				/* out */ int& ro, int& re, double& delta_lo, double& delta_le,
				/* inout */ x_line<int>& dup_o, x_line<int>& dup_e, bool rollback )
{
	// second case: determine which gradients corresponding to vectors from R will first reach -e or +e
	// returns rest vector index and associated delta_lambda_k value
	ro = re = -1;
	delta_lo = delta_le = 0;
	for (int i = 1 ; i < gamma_k.len () ; i ++)
	{
		int iminus = i - 1;
		x_elem& r = r_vect[iminus];
		double h = calc_h (r.idx, &uv);
		if (r.type == x_elem::ZERO)
		{
			
			double se = incremental && gamma_k[i] < 0 || !incremental && gamma_k[i] > 0 ? -1 : 1;
			double delta = (se * eps - h) / gamma_k[i];
			INFO (4, "    %d[%g] lo: lambda: %g h_k: %g gamma_k: %g delta: %g\n",
				r.idx, probl.y[r.idx], r.lambda, h, gamma_k[i], delta);
			if (	!rollback &&
				(IS_CONST (h, eps) && se == 1 ||
				IS_CONST (h, -eps) && se == -1 ) )
				continue;

			//double delta = (se * eps - h) / gamma_k[i];

			// debug purposes
			//if (calc_h (r, &uv) < 0)
			//{
			//	TRACE ("[trouble] %g is nonpositive, but (%d) is O vector\n", calc_h (r, &uv), r);
			//	print_state (&uv);
			//	exit (1);
			//}
			//INFO (4, "    %d[%g] lo: lambda: %g h_k: %g gamma_k: %g delta: %g\n",
			//	r.idx, probl.y[r.idx], r.lambda, h, gamma_k[i], delta);

			if (ro < 0) {
				ro = iminus;
				delta_lo = delta;
			} else if (IS_ZERO (delta - delta_lo)) {
				// duplicate delta found (seldom)
				if (dup_o.len () == 0) {
					dup_o.add (ro);
					dup_o.add (iminus);
				} else if (dup_o[0] == ro) {
					dup_o.add (iminus);
				} else {
					while (dup_o.len () > 0)
						dup_o.remove (dup_o.len () - 1);
					dup_o.add (ro);
					dup_o.add (iminus);
				}
			} else if (incremental && delta < delta_lo || !incremental && delta > delta_lo) {
				ro = iminus;
				delta_lo = delta;
			}
		} else {
			// bounded, r_vect[iminus].type == x_elem::BOUNDED
			double sl = r.lambda < 0 ? -1 : 1;
			double sg = gamma_k[i] < 0 ? -1 : 1;
			if (incremental && sl == sg || !incremental && sl != sg)
			{
				double delta = (- sl * eps - h) / gamma_k[i];

				INFO (4, "    %d[%g] le: lambda: %g h_k: %g gamma_k: %g delta: %g\n",
					r.idx, probl.y[r.idx], r.lambda, h, gamma_k[i], delta);

				if (re < 0) {
					re = iminus;
					delta_le = delta;
				} else if (IS_ZERO (delta - delta_le)) {
					// duplicate delta found (seldom)
					if (dup_e.len () == 0) {
						dup_e.add (re);
						dup_e.add (iminus);
					} else if (dup_e[0] == re) {
						dup_e.add (iminus);
					} else {
						while (dup_e.len () > 0)
							dup_e.remove (dup_e.len () - 1);
						dup_e.add (re);
						dup_e.add (iminus);
					}
				} else if (incremental && delta < delta_le || !incremental && delta > delta_le) {
					re = iminus;
					delta_le = delta;
				}
			}
		}
	}
}


void x_svm_r::limit_third_extra (	/* in */ bool incremental, int k, x_elem& uv, x_line<double>& gamma_k,
					/* out */ bool& enabled, double& delta_lc )
{
	// third case: determine largest amount for h_c to reach -e or +e
	// makes sense only for gamma_k[0] and epsilon having different signs
	// for the incremental case, or vice-versa for decremental;
	// otherwise, an additional decrease it does not matter
	// usually, an increase/decrease means error vector gets 'learned'
	// returns the associated delta_lambda_k
	double h = calc_h (k, &uv);
	double se = h < 0 ? -1 : 1;
	double sg = gamma_k[0] < 0 ? -1 : 1;
	if (incremental && se != sg || !incremental && se == sg)
	{
		enabled = true;
		delta_lc = (se * eps - h) / gamma_k[0];
	} else {
		enabled = false;
		delta_lc = 0;
	}
}

void x_svm_r::find_min_delta (	/* in */ bool enable[], double delta_l[], int len,
				/* out */ int& min_idx )
{
	// find the minimum from the five elements
	min_idx = -1;
	for (int i = 0 ; i < len ; i ++)
		if (enable[i])
		{
			if (min_idx < 0)
				min_idx = i;
			else if (delta_l[i] < delta_l[min_idx] && !IS_ZERO (delta_l[i] - delta_l[min_idx]))
				min_idx = i;
		}
}


void x_svm_r::find_max_delta (	/* in */ bool enable[], double delta_l[], int len,
				/* out */ int& max_idx )
{
	// find the minimum from the five elements
	max_idx = -1;
	for (int i = 0 ; i < len ; i ++)
		if (enable[i])
		{
			if (max_idx < 0)
				max_idx = i;
			else if (delta_l[i] > delta_l[max_idx] && !IS_ZERO (delta_l[i] > delta_l[max_idx]))
				max_idx = i;
		}
}


void x_svm_r::update_margins ( /* in */ double dlc, x_line<double>& beta_k )
{
	// update w0 and lambda_s accordingly
	x_line<double> delta;
	delta.ensure (beta_k.len ());

	for (int i = 0 ; i < delta.len () ; i ++)
		delta[i] = beta_k[i] * dlc;

	w0 += delta[0];
	for (int i = 0 ; i < s_vect.len () ; i ++)
		s_vect[i].lambda += delta[i + 1];
}


inline void x_svm_r::print_state (/* unlearned vector */ x_elem * uv)
{
	double sum = 0;
	double mse = 0;
	double mae = 0;
	int n = 0;

	printf ("C: %g sigma: %g\n", param.C, param.sigma);
	INFO (3, "gradients for SVs[%d]:\n", s_vect.len ());
	for (int i = 0 ; i < s_vect.len () ; i ++)
	{
		int idx = s_vect[i].idx;
		double h = calc_h (idx, uv);
		INFO (3, "  %d[%g]: %g - ", idx, probl.y [idx], h);
		INFO (3, "%s", s_vect[i].type == x_elem::BOUNDED ? "bounded" : 
				(s_vect[i].type == x_elem::ZERO ? "zero" : "unbounded"));
		INFO (3, "( %g )\n", s_vect[i].lambda);

		if (s_vect[i].type != x_elem::UNBOUNDED)
		{
			TRACE ("SV: %d should always be unbounded!\n", s_vect[i].idx);
			exit (1);
		}
		if (!IS_CONST2 (h, - eps) && !IS_CONST2 (h, eps))
		{
			TRACE ("unbounded SV: %d should have gradient %g %g or %g!\n", s_vect[i].idx, h, - eps, eps);
			//exit (1);
		}
		
		sum += s_vect[i].lambda;
		mse += h * h;
		mae += abs (h);
		n ++;
	}
	
	INFO (3, "gradients for Rest[%d]:\n", r_vect.len ());
	int bound = 0;
	int accuracy = 0;
	for (int i = 0 ; i < r_vect.len () ; i ++)
	{
		int idx = r_vect[i].idx;
		double h = calc_h (idx, uv);
		INFO (3, "  %d[%g]: %g - ", idx, probl.y [idx], h);
		INFO (3, "%s", r_vect[i].type == x_elem::BOUNDED ? "bounded" : 
				(r_vect[i].type == x_elem::ZERO ? "zero" : "unbounded"));
		INFO (3, "( %g )\n", r_vect[i].lambda);
	
		if (r_vect[i].type == x_elem::BOUNDED && !IS_CONST2 (abs(r_vect[i].lambda), param.C))
		{
			TRACE ("bounded vector: %d should have %g equals %g !\n", 
				r_vect[i].idx, r_vect[i].lambda, sgn(r_vect[i].lambda) * param.C);
			exit (1);
		}
		if (r_vect[i].type == x_elem::BOUNDED && r_vect[i].lambda < 0 && h < 0)
		{
			TRACE ("bounded vector: %d lambda: %g should have gradient %g > 0 !\n",
				r_vect[i].idx, r_vect[i].lambda, h);
			exit (1);
		}
		if (r_vect[i].type == x_elem::BOUNDED && r_vect[i].lambda > 0 && h > 0)
		{
			TRACE ("bounded vector: %d lambda: %g should have gradient %g < 0 !\n",
				r_vect[i].idx, r_vect[i].lambda, h);
			exit (1);
		}

		if (r_vect[i].type == x_elem::ZERO && !IS_CONST2 (h, eps) && !IS_CONST2 (h, -eps) && (h < - eps || h > eps))
		{
			TRACE ("zero vector: %d should have gradient %g in tube!\n", r_vect[i].idx, h);
			//exit (1);
		}

		if (r_vect[i].type == x_elem::BOUNDED)
		{
			bound ++;
			sum += r_vect[i].lambda;
		}
		if (r_vect[i].type == x_elem::ZERO)
			accuracy ++;

		mse += h * h;
		mae += abs (h);
		n ++;
	}
	INFO (3, "KKT sum: %g\n", sum);
	INFO (3, "Others / Rest: %d/%d (%3.2f%%) - accuracy: %d/%d (%3.2f%%)\n", r_vect.len () - bound, r_vect.len (), 
			((float)(r_vect.len () - bound) * 100) / r_vect.len (),
			accuracy + s_vect.len (), r_vect.len () + s_vect.len (),
			((float)(accuracy + s_vect.len ()) * 100) / (r_vect.len () + s_vect.len ())
	);
	INFO (3, "train MSE: %g MAE: %g\n", mse / n, mae / n);
}


void x_svm_r::check_pq_consistency (void)
{
	// check consistence of P and Q
	INFO (1, "P:\n");
	p.print ();
	INFO (1, "Q:\n");
	q.print ();
	INFO (1, "P x Q product:\n");
	x_matrix<double> o;
	o.ensure (p.len ());
	x_product<double>::mul_mm (p, q, o);
	o.print ();
}


int x_svm_r::check_duplicate (int v, bool inSupport)
{
	if (inSupport)
		for (int i = 0 ; i < s_vect.len () ; i ++)
		{
			int u = s_vect[i].idx;
			bool duplicate = true;
			for (int j = 0 ; j < s_vect.len () ; j ++)
			{
				int k = s_vect[j].idx;
				if ((this->*get_q) (u, k) != (this->*get_q) (v, k))
				{
					duplicate = false;
					break;
				}
			}
	
			if (duplicate && u != v)
				return u;
		}
	else
		for (int i = 0 ; i < r_vect.len () ; i ++)
		{
			int u = r_vect[i].idx;
			bool duplicate = true;
			for (int j = 0 ; j < r_vect.len () ; j ++)
			{
				int k = r_vect[j].idx;
				if ((this->*get_q) (u, k) != (this->*get_q) (v, k))
				{
					duplicate = false;
					break;
				}
			}
	
			if (duplicate && u != v)
				return u;
		}

	return -1;
}


void x_svm_r::rollback (	/* [in] learned pattern */ int k,
		/* [in] delta_lc */ double lc,
		/* [in] eta_k */ x_line<double>& eta_k,
		/* [in] beta_k */ x_line<double>& beta_k,
		/* [in] gamma_k */ x_line<double>& gamma_k )
{
	// decrement lc to zero
	bool update_eta_beta_gamma = true;
	bool dir = lc > 0 ? false : true;
	while (lc != 0) {
		INFO (1, "===[*** - k: %4d ]======[cycle - lc: %g ]=unlearn======================\n", k, lc);

		x_elem uv = {k, lc, x_elem::UNBOUNDED};
		if (g_dbgLevel > 2)
			print_state (&uv);

		// update eta_k, beta_k, gamma_h
		if (update_eta_beta_gamma)
		{
			calculate_eta (k, eta_k);
			calculate_beta (eta_k, beta_k);
			calculate_gamma (k, beta_k, gamma_k);
			update_eta_beta_gamma = false;
		}

		// allocate space to values
		bool enable[5];
		double delta_l[5];
		x_line<int> dup[4];

		// first case: determine which vectors from S are first to reaching limits
		int sm = -1, sp = -1;
		limit_first_unbounded (dir, uv, beta_k, sm, sp, /*lm*/ delta_l[0], /*lp*/ delta_l[1], dup[0], dup[1], true);
		enable[0] = sm < 0 ? false : true;
		enable[1] = sp < 0 ? false : true;
		INFO (3, "delta_lm: %g (%d) ->%s\n", delta_l[0], sm < 0 ? -1 : s_vect[sm].idx, enable[0] ? "true" : "false");
		INFO (3, "delta_lp: %g (%d) ->%s\n", delta_l[1], sp < 0 ? -1 : s_vect[sp].idx, enable[1] ? "true" : "false");

		// second case: determine which gradients corresponding to vectors from R will first reach zero
		int ro = -1, re = -1;
		limit_second_rest (dir, uv, gamma_k, ro, re, /*lo*/ delta_l[2], /*le*/ delta_l[3], dup[2], dup[3], true);
		enable[2] = ro < 0 ? false : true;
		enable[3] = re < 0 ? false : true;
		INFO (3, "delta_lo: %g (%d) ->%s\n", delta_l[2], ro < 0 ? -1 : r_vect[ro].idx, enable[2] ? "true" : "false");
		INFO (3, "delta_le: %g (%d) ->%s\n", delta_l[3], re < 0 ? -1 : r_vect[re].idx, enable[3] ? "true" : "false");

		// third case: lambda_c should not underrun 0
		enable[4] = true;
		delta_l[4] = -lc;
		INFO (2, "delta_lc: %g ->%s\n", delta_l[4], enable[4] ? "true" : "false");

		// find the maximum from the five elements
		int m_idx = -1;
		if (dir)
			find_min_delta (enable, delta_l, 5, m_idx);
		else 
			find_max_delta (enable, delta_l, 5, m_idx);
		if (dir && delta_l[m_idx] < 0 || !dir && delta_l[m_idx] > 0)
		{
			// just ignore it and try again
			//TRACE ("positive delta found: %g\n", delta_l[max_idx]);
			enable[m_idx] = false;

			m_idx = -1;
			if (dir)
				find_min_delta (enable, delta_l, 5, m_idx);
			else
				find_max_delta (enable, delta_l, 5, m_idx);
			if (dir && delta_l[m_idx] < 0 || !dir && delta_l[m_idx] > 0)
			{
				// this means really trouble
				TRACE ("positive/negative delta found again: %g\n", delta_l[m_idx]);
				exit (1);
			}
		}

		// update w0 and lambda_s accordingly
		{
			double dlc = delta_l[m_idx];
			INFO (2, "dlc: %g\n", dlc);

			lc += dlc;
			uv.lambda = lc;

			update_margins (dlc, beta_k);
		}
			
		// determine which vector is going to migrate between sets
		update_eta_beta_gamma = true;
		switch (m_idx)
		{
		case 0:
			// k in S, sm
			// take out vector sm from S and put it to R (O)
			INFO (2, "--> %d (S) migrates to O\n", s_vect[sm].idx); 
			{
				if (!IS_ZERO2 (s_vect[sm].lambda))
				{
					TRACE ("lambda_sm: %g should be zero!\n", s_vect[sm].lambda);
					exit (1);
				}
				x_elem l = {s_vect[sm].idx, 0, x_elem::ZERO};
				s_vect.remove (sm);
				r_vect.add (l);

				// update P and Q
				contract_pq (sm);
			}
			break;
		case 1:
			// k in S, sp
			// take out vector sp from S and put it to R (E)
			INFO (2, "--> %d (S) migrates to E\n", s_vect[sp].idx);
			{
				if (!IS_CONST2 (s_vect[sp].lambda, param.C) && !IS_CONST2 (s_vect[sp].lambda, -param.C))
				{
					TRACE ("lambda_sp: %g should be +/-%g!\n", s_vect[sp].lambda, param.C);
					exit (1);
				}
				x_elem l = {s_vect[sp].idx, (s_vect[sp].lambda < 0 ? -1 : 1) * param.C, x_elem::BOUNDED};
				s_vect.remove (sp);
				r_vect.add (l);

				// update P and Q
				contract_pq (sp);
			}
			break;
		case 2:
			// k in R (O), ro
			// we have a support vector candidate, transferred from O to S
			INFO (2, "--> %d (O) migrates to S\n", r_vect[ro].idx);
			{
				int idx_ro = r_vect[ro].idx;
				double h = calc_h (r_vect[ro].idx, &uv);
				if (!IS_CONST2 (h, eps) && !IS_CONST2 (h, -eps))
				{
					TRACE ("gradient h_ro: %g should be +/-%g!\n", h, eps);
					exit (1);
				}
				if (!IS_ZERO (r_vect[ro].lambda))
				{
					TRACE ("lambda_ro: %g should be zero!\n", r_vect[ro].lambda);
					exit (1);
				}
				x_elem l = {r_vect[ro].idx, 0, x_elem::UNBOUNDED};
				r_vect.remove (ro);
				s_vect.add (l);

				// update P and Q
				expand_pq (l.idx);

				if (dup[2].len () && ro == dup[2][0])
				{
					for (int i = 1 ; i < dup[2].len () ; i ++)
					{
						// we already removed some vector(s)
						int id = dup[2][i] - i;
						INFO (1, "--> (rollback) duplicate result found for ro: %d\n", idx_ro);

						// check for vector equality
						int idx_id = r_vect[id].idx;
						bool matches = true;
						for (int j = 0 ; j < s_vect.len () ; j ++)
							if (	(this->*get_q) (idx_ro, s_vect[j].idx) !=
								(this->*get_q) (idx_id, s_vect[j].idx))
							{
								matches = false;
								break;
							}
						if (matches)
						{
							INFO (1, "--> %d matches with %d, last one discarded\n",
									idx_ro, idx_id);
							r_vect.remove (id);
							continue;
						}

						double h = calc_h (r_vect[id].idx, &uv);
						INFO (2, "--> %d (O) migrates to S\n", r_vect[id].idx);
						if (!IS_CONST2 (h, eps) && !IS_CONST2 (h, -eps))
						{
							TRACE ("gradient h_ro(%d): %g should be +/-%g!\n",
								r_vect[id].idx, h, eps);
							exit (1);
						}
						if (!IS_ZERO2 (r_vect[id].lambda))
						{
							TRACE ("lambda_ro(%d): %g should be zero!\n",
								r_vect[id].idx, r_vect[id].lambda);
							exit (1);
						}
						x_elem l = {r_vect[id].idx, 0, x_elem::UNBOUNDED};
						r_vect.remove (id);
						s_vect.add (l);
		
						// update P and Q
						expand_pq (l.idx);
					}
				}
			}
			break;
		case 3:
			// k in R (E), re
			// we have a support vector candidate, transferred from E to S
			INFO (2, "--> %d (E) migrates to S\n", r_vect[re].idx);
			{
				double h = calc_h (r_vect[re].idx, &uv);
				if (!IS_CONST2 (h, eps) && !IS_CONST2 (h, -eps))
				{
					TRACE ("gradient h_re: %g should be +/-%g!\n", h, eps);
					exit (1);
				}
				if (!IS_CONST2 (r_vect[re].lambda, param.C) && !IS_CONST2 (r_vect[re].lambda, -param.C))
				{
					TRACE ("lambda_re: %g should be +/-%g!\n", r_vect[re].lambda, param.C);
					exit (1);
				}
				x_elem l = {r_vect[re].idx, (r_vect[re].lambda < 0 ? -1 : 1) * param.C, x_elem::UNBOUNDED};
				r_vect.remove (re);
				s_vect.add (l);

				// update P and Q
				expand_pq (l.idx);
			}
			break;
		case 4:
			// all deltas are greater than maximum increment before reaching 0
			// so k goes to E
			INFO (1, "--> %d (U) unlearned\n", k);
			break;
		default:
			TRACE ("should not reach default!\n");
			exit (1);
		}
	}
}


int x_svm_r::pattern_learn (	/* [in] cycle number */ int incr,
				/* [in] (yet) unlearned pattern */ int k,
				/* [in] eta_k */ x_line<double>& eta_k,
				/* [in] beta_k */ x_line<double>& beta_k,
				/* [in] gamma_k */ x_line<double>& gamma_k )
{
	// check whether this can be put to O (Rest)
	{
		double h = calc_h (k);

		if (g_dbgLevel > 2)
		{
			int dsv = check_duplicate (k);
			if (dsv > -1)
			{
				INFO (3, "--> %d (U) duplicate contribution with SV %d, informing\n", k, dsv);
	// 			exit(1);
			}
			dsv = check_duplicate (k, false);
			if (dsv > -1)
			{
				INFO (3, "--> %d (U) duplicate contribution with RV %d, informing\n", k, dsv);
	// 			exit(1);
			}
		}

		if (IS_CONST2(h, eps) || IS_CONST2(h, -eps))
		{
			// check for duplicate vector
			int dsv = check_duplicate (k);
			if (dsv > -1)
			{
				INFO (1, "--> %d (U) duplicate contribution with SV %d, skipped\n", k, dsv);
				return 0;
			}

			INFO (2, "--> %d (U) migrates to S, gradient: %g\n", k, h);
			x_elem l = {k, 0, x_elem::UNBOUNDED};
			s_vect.add (l);

			// update P and Q
			expand_pq (l.idx);
			return 0;
		} else if (h > - eps && h < eps) {
			INFO (2, "--> %d (U) migrates to O, gradient: %g\n", k, h);
			x_elem l = {k, 0, x_elem::ZERO};
			r_vect.add (l);
			
			return 0;
		}

		// this vector has negative gradient
		INFO (2, "vector %d has h: %g\n", k, h);
	}

	// start incrementing lc, from zero to C
	double lc = 0;
	bool update_eta_beta_gamma = true;
	bool dir = true;
	do {
		// update eta_k, beta_k, gamma_h
		if (update_eta_beta_gamma)
		{
			calculate_eta (k, eta_k);
			calculate_beta (eta_k, beta_k);
			calculate_gamma (k, beta_k, gamma_k);
			update_eta_beta_gamma = false;
		}

		if (lc == 0)
		{
			double h = calc_h (k);
			dir = h > 0 && gamma_k[0] < 0 || h < 0 && gamma_k[0] > 0 ? true : false;
		}

		INFO (1, "===[%d - k: %4d ]======[cycle - lc: %g dir: %s ]==============================\n",
			incr, k, lc, dir ? "increment" : "decrement");
		x_elem uv = {k, lc, x_elem::UNBOUNDED};
		INFO (4, "h: %g\n", calc_h (k, &uv));
		if (g_dbgLevel > 2) 
			print_state (&uv);

		// allocate space to values
		bool enable[6];
		double delta_l[6];

		// store positions of duplicate minimas (very rare, but possible - found one)
		x_line<int> dup[4];

		// first case: determine which vectors from S are first to reaching limits
		int sm = -1, sp = -1;
		limit_first_unbounded (dir, uv, beta_k, sm, sp, /* lm */ delta_l[0], /* lp */ delta_l[1], dup[0], dup[1]);
		enable[0] = sm < 0 ? false : true;
		enable[1] = sp < 0 ? false : true;
		INFO (3, "delta_lm: %g (%d) ->%s\n", delta_l[0], sm < 0 ? -1 : s_vect[sm].idx, enable[0] ? "true" : "false");
		INFO (3, "delta_lp: %g (%d) ->%s\n", delta_l[1], sp < 0 ? -1 : s_vect[sp].idx, enable[1] ? "true" : "false");

		// second case: determine which gradients corresponding to vectors from R will first reach zero
		int ro = -1, re = -1;
		limit_second_rest (dir, uv, gamma_k, ro, re, /* lo */delta_l[2], /* le */ delta_l[3], dup[2], dup[3]);
		enable[2] = ro < 0 ? false : true;
		enable[3] = re < 0 ? false : true;
		INFO (3, "delta_lo: %g (%d) ->%s\n", delta_l[2], ro < 0 ? -1 : r_vect[ro].idx, enable[2] ? "true" : "false");
		INFO (3, "delta_le: %g (%d) ->%s\n", delta_l[3], re < 0 ? -1 : r_vect[re].idx, enable[3] ? "true" : "false");

		// third case: determine largest amount for h_k to reach zero
		// makes sense for gamma_k[0] > 0
		limit_third_extra (dir, k, uv, gamma_k, enable[4], delta_l[4]);
		INFO (3, "delta_lk: %g (%d) ->%s\n", delta_l[4], k, enable[4] ? "true" : "false");

		// fourth case: lambda_c should not overflow C
		enable[5] = true;
		delta_l[5] = dir ? param.C - lc : - param.C - lc;
		INFO (3, "delta_lc: %g ->%s\n", delta_l[5], enable[5] ? "true" : "false");

		// find the minimum from the five elements
		int m_idx = -1;
		if (dir)
			find_min_delta (enable, delta_l, 6, m_idx);
		else
			find_max_delta (enable, delta_l, 6, m_idx);
		if (dir && delta_l[m_idx] < 0 || !dir && delta_l[m_idx] > 0)
		{
			TRACE ("different delta found: %g\n", delta_l[m_idx]);
			exit (1);
		}

		for (int i = 0 ; i < 4 ; i ++)
		{
			if (!enable[i])
				continue;

			int idx_id = 0;
			switch (i)
			{
				case 0:
					idx_id = s_vect[sm].idx;
					break;
				case 1:
					idx_id = s_vect[sp].idx;
					break;
				case 2:
					idx_id = r_vect[ro].idx;
					break;
				case 3:
					idx_id = r_vect[re].idx;
					break;
			}

			if (	IS_ZERO (delta_l[i] - delta_l[4]) ||
				calc_h (k) + probl.y[k] == calc_h (idx_id) + probl.y[idx_id] )
			{
				// check for duplicate vector trying to exit / enter S, with current
				bool matches = true;
				for (int j = 0 ; j < s_vect.len () ; j ++)
					if (	(this->*get_q) (k, s_vect[j].idx) != 
						(this->*get_q) (idx_id, s_vect[j].idx))
					{
						matches = false;
						break;
					}
				if (matches)
				{
					INFO (1, "--> %d matches with %d, discarding last one\n", idx_id, k);
					rollback (k, lc, eta_k, beta_k, gamma_k);
					lc = 0;
					return 0;
				}
			}
		}

		if (g_dbgLevel > 3)
			for (int i = 0 ; i < s_vect.len () ; i ++)
			{
				INFO (4, "(%d) lambda: %g h_s: %g\n", 
					s_vect[i].idx, s_vect[i].lambda, calc_h (s_vect[i].idx, &uv));
			}
		// update w0 and lambda_s accordingly
		{
			double dlc = delta_l[m_idx];
			INFO (2, "dlc: %g\n", dlc);

			lc += dlc;
			uv.lambda = lc;

			update_margins (dlc, beta_k);
		}
		if (g_dbgLevel > 3)
			for (int i = 0 ; i < s_vect.len () ; i ++)
			{
				INFO (4, "(%d) new_lambda: %g new_h_s: %g\n", 
					s_vect[i].idx, s_vect[i].lambda, calc_h (s_vect[i].idx, &uv));
			}

		// determine which vector is going to migrate between sets
		update_eta_beta_gamma = true;
		switch (m_idx)
		{
		case 0:
			// k in S, sm
			// take out vector sm from S and put it to R (O)
			INFO (2, "--> %d (S) migrates to O\n", s_vect[sm].idx); 
			{
				int idx_sm = s_vect[sm].idx;
				if (!IS_ZERO2 (s_vect[sm].lambda))
				{
					TRACE ("lambda_sm: %g should be zero!\n", s_vect[sm].lambda);
					exit (1);
				}
				x_elem l = {idx_sm, 0, x_elem::ZERO};
				s_vect.remove (sm);
				r_vect.add (l);

				// update P and Q
				contract_pq (sm);

				if (dup[0].len () && sm == dup[0][0])
				{
					for (int i = 1 ; i < dup[0].len () ; i ++)
					{
						// one vector already removed from S
						int id = dup[0][i] - i;
						INFO (2, "--> duplicate result found for sm: %d\n", idx_sm);

						INFO (2, "--> %d (S) migrates to O\n", s_vect[id].idx);
						if (!IS_ZERO2 (s_vect[id].lambda))
						{
							TRACE ("lambda_smid: %g should be zero!\n", s_vect[id].lambda);
							exit (1);
						}
						x_elem l = {s_vect[id].idx, 0, x_elem::ZERO};
						s_vect.remove (id);
						r_vect.add (l);
		
						// update P and Q
						contract_pq (id);
					}
				}
			}
			break;
		case 1:
			// k in S, sp
			// take out vector sp from S and put it to R (E)
			INFO (2, "--> %d (S) migrates to E\n", s_vect[sp].idx);
			{
				int idx_sp = s_vect[sp].idx;
				if (!IS_CONST2(s_vect[sp].lambda, param.C) && !IS_CONST2(s_vect[sp].lambda, -param.C))
				{
					TRACE ("lambda_sp: %g should be +/-%g!\n", s_vect[sp].lambda, param.C);
					exit (1);
				}
				x_elem l = {idx_sp, (s_vect[sp].lambda < 0 ? -1 : 1) * param.C, x_elem::BOUNDED};
				s_vect.remove (sp);
				r_vect.add (l);

				// update P and Q
				contract_pq (sp);

				if (dup[1].len () && sp == dup[1][0])
				{
					for (int i = 1 ; i < dup[1].len () ; i ++)
					{
						// one vector already removed from S
						int id = dup[1][i] - i;
						INFO (2, "--> duplicate result found for sp: %d\n", idx_sp);

						INFO (2, "--> %d (S) migrates to E\n", s_vect[id].idx);
						if (	!IS_CONST2(s_vect[id].lambda, param.C) && 
							!IS_CONST2(s_vect[id].lambda, -param.C) )
						{
							TRACE ("lambda_spid: %g should be +/-%g!\n",
								s_vect[id].lambda, param.C);
							exit (1);
						}
						x_elem l = {s_vect[id].idx, (s_vect[id].lambda < 0 ? -1 : 1) * param.C,
							x_elem::BOUNDED};
						s_vect.remove (id);
						r_vect.add (l);
		
						// update P and Q
						contract_pq (id);
					}
				}
			}
			break;
		case 2:
			// k in R (O), ro
			// we have a support vector candidate, transferred from O to S
			{
				int idx_ro = r_vect[ro].idx;
				double h = calc_h (r_vect[ro].idx, &uv);
				INFO (2, "--> %d (O) migrates to S\n", r_vect[ro].idx);
				if (!IS_CONST2 (h, eps) && !IS_CONST2 (h, -eps))
				{
					TRACE ("gradient h_ro: %g should be +/-%g!\n", h, eps);
					exit (1);
				}
				if (!IS_ZERO2 (r_vect[ro].lambda))
				{
					TRACE ("lambda_ro: %g should be zero!\n", r_vect[ro].lambda);
					exit (1);
				}
				x_elem l = {r_vect[ro].idx, 0, x_elem::UNBOUNDED};
				r_vect.remove (ro);
				s_vect.add (l);

				// update P and Q
				expand_pq (l.idx);

				if (dup[2].len () && ro == dup[2][0])
				{
					for (int i = 1 ; i < dup[2].len () ; i ++)
					{
						// we already removed some vector(s)
						int id = dup[2][i] - i;
						INFO (2, "--> duplicate result found for ro: %d\n", idx_ro);

						// check for vector equality
						int idx_id = r_vect[id].idx;
						bool matches = true;
						for (int j = 0 ; j < s_vect.len () ; j ++)
							if (	(this->*get_q) (idx_ro, s_vect[j].idx) != 
								(this->*get_q) (idx_id, s_vect[j].idx))
							{
								matches = false;
								break;
							}
						if (matches)
						{
							INFO (1, "--> %d matches with %d, last one discarded\n",
									idx_ro, idx_id);
							r_vect.remove (id);
							continue;
						}

						double h = calc_h (r_vect[id].idx, &uv);
						INFO (2, "--> %d (O) migrates to S\n", r_vect[id].idx);
						if (!IS_CONST2 (h, eps) && !IS_CONST2 (h, -eps))
						{
							TRACE ("gradient h_ro(%d): %g should be +/- %g!\n",
								r_vect[id].idx, h, eps);
							exit (1);
						}
						if (!IS_ZERO2 (r_vect[id].lambda))
						{
							TRACE ("lambda_ro(%d): %g should be zero!\n",
								r_vect[id].idx, r_vect[id].lambda);
							exit (1);
						}
						x_elem l = {r_vect[id].idx, 0, x_elem::UNBOUNDED};
						r_vect.remove (id);
						s_vect.add (l);
		
						// update P and Q
						expand_pq (l.idx);
					}
				}
			}
			break;
		case 3:
			// k in R (E), re
			// we have a support vector candidate, transferred from E to S
			{
				int idx_re = r_vect[re].idx;
				double h = calc_h (r_vect[re].idx, &uv);
				INFO (2, "--> %d (E) migrates to S\n", r_vect[re].idx);
				if (!IS_CONST2 (h, eps) && !IS_CONST2 (h, -eps))
				{
					TRACE ("gradient h_re: %g should be +/-%g!\n", h, eps);
					exit (1);
				}
				if (!IS_CONST2 (r_vect[re].lambda, param.C) && !IS_CONST2 (r_vect[re].lambda, -param.C))
				{
					TRACE ("lambda_re: %g should be +/-%g!\n", r_vect[re].lambda, param.C);
					exit (1);
				}

				x_elem l = {r_vect[re].idx, (r_vect[re].lambda < 0 ? -1 : 1) * param.C, x_elem::UNBOUNDED};
				r_vect.remove (re);
				s_vect.add (l);

				// update P and Q
				expand_pq (l.idx);

				if (dup[3].len () && re == dup[3][0])
				{
					for (int i = 1 ; i < dup[3].len () ; i++)
					{
						// we already removed some vector(s)
						int id = dup[3][i] - i;
						INFO (2, "--> duplicate result found for re: %d\n", idx_re);

						// check for vector equality
						int idx_id = r_vect[id].idx;
						bool matches = true;
						for (int j = 0 ; j < s_vect.len () ; j ++)
							if (	(this->*get_q) (idx_re, s_vect[j].idx) !=
								(this->*get_q) (idx_id, s_vect[j].idx))
							{
								matches = false;
								break;
							}
						if (matches)
						{
							INFO (1, "--> %d matches with %d, discarding last one\n",
									idx_re, idx_id);
							rollback (k, lc, eta_k, beta_k, gamma_k);
							lc = r_vect[id].lambda;
							int idx = r_vect[id].idx;
							INFO (1, "--> removing %d from Rest\n", idx);
							r_vect.remove (id);
							rollback (idx, lc, eta_k, beta_k, gamma_k);
							lc = 0;
							continue;
						}
 
						double h = calc_h (r_vect[id].idx, &uv);
						INFO (2, "--> %d (E) migrates to S\n", r_vect[id].idx); 
						if (!IS_CONST2 (h, eps) && !IS_CONST2 (h, -eps))
						{
							TRACE ("gradient h_re(%d): %g should be +/-%g!\n",
								r_vect[id].idx, h, eps);
							exit (1);
						}
						if (	!IS_CONST2 (r_vect[id].lambda, param.C) && 
							!IS_CONST2 (r_vect[id].lambda, -param.C) )
						{
							TRACE ("lambda_re(%d): %g should be +/-%g!\n",
								r_vect[id].idx, r_vect[id].lambda, param.C);
							exit (1);
						}
						x_elem l = {r_vect[id].idx, (r_vect[id].lambda < 0 ? -1 : 1) * param.C,
							x_elem::UNBOUNDED};
						r_vect.remove (id);
						s_vect.add (l);
	
						// update P and Q
						expand_pq (l.idx);
					}
				}
			}
			break;
		case 4:
			// all remains, new vector contribution h_k will reach zero first
			// depending on lambda_c, it will join S or E (now considered for S)
			INFO (2, "--> %d (U) migrates to S\n", k);
			{
				double h = calc_h (k, &uv);
				if (!IS_CONST2 (h, -eps) && !IS_CONST2 (h, eps))
				{
					TRACE ("gradient h_k: %g should be +/-%g!\n", h, eps);
					exit (1);
				}
				if (IS_CONST2 (lc, param.C) || IS_CONST2 (lc, -param.C) || lc > param.C || lc < - param.C)
				{
					TRACE ("lambda_rk: %g should be in tube +/-%g!\n", lc, param.C);
					exit (1);
				}
				x_elem l = {k, lc, x_elem::UNBOUNDED};
				s_vect.add (l);

				// update P and Q
				expand_pq (l.idx);

				// pattern learned, go to the next one
				lc = dir ? param.C : -param.C;
			}
			break;
		case 5:
			// all deltas are greater than maximum increment before reaching C
			// so k goes to E
			INFO (2, "--> %d (U) migrates to E\n", k);
			{
				//double h = calc_h (k, &uv);
				//if (dir && h >= 0) {
				//	TRACE ("gradient h_k: %g should be negative!\n", h);
				//	exit (1);
				//} else if (!dir && h <= 0) {
				//	TRACE ("gradient h_k: %g should be positive!\n", h);
				//	exit (1);
				//}
				if (!IS_CONST2 (lc, param.C) && !IS_CONST2 (lc, -param.C))
				{
					TRACE ("lambda_rc: %g should equals +/-%g!\n", lc, param.C);
					exit (1);
				}
				lc = dir ? param.C : -param.C;
				x_elem l = {k, lc, x_elem::BOUNDED};
				r_vect.add (l);
				update_eta_beta_gamma = false;
			}
			break;
		default:
			TRACE ("should not reach default!\n");
			exit (1);
		}

		if (s_vect.len () == 0)
		{
			TRACE ("no SVs left in solution, try again");
			return -1;
		}
	} while (dir && lc < param.C || !dir && lc > - param.C);

	return 0;
}


void x_svm_r::pattern_unlearn (	/* [in] SV / RV */ bool isSupport,
				/* [in] pattern to be unlearned */ int k,
				/* [in] eta_k */ x_line<double>& eta_k,
				/* [in] beta_k */ x_line<double>& beta_k,
				/* [in] gamma_k */ x_line<double>& gamma_k )
{
	INFO (1, "===[unlearn - k: %4d ]======[ current state ]==============================\n", k);
	print_state ();

	if (isSupport)
	{
		// check for existence in S
		if (k < 0 || k >= s_vect.len ())
		{
			TRACE ("invalid index given for SV: %d / %d", k, s_vect.len () - 1);
			exit (1);
		}

		// remove element from S
		x_elem uv = { s_vect[k].idx, s_vect[k].lambda, x_elem::UNBOUNDED };
		s_vect.remove (k);
		contract_pq (k);

		// now try to unlearn it
		rollback (uv.idx, uv.lambda, eta_k, beta_k, gamma_k);
	} else {
		// check for existence in R
		if (k < 0 || k >= r_vect.len ())
		{
			TRACE ("invalid index given for SV: %d / %d", k, s_vect.len () - 1);
			exit (1);
		}

		// remove element from R
		x_elem uv = { r_vect[k].idx, r_vect[k].lambda, r_vect[k].type };
		r_vect.remove (k);
	
		// now try to unlearn it (if E)
		if (uv.type == x_elem::BOUNDED)
			rollback (uv.idx, uv.lambda, eta_k, beta_k, gamma_k);
	}

	print_state ();
}


void x_svm_r::pattern_unlearn (	/* [in] pattern to be unlearned in set */ int k,
				/* [in] eta_k */ x_line<double>& eta_k,
				/* [in] beta_k */ x_line<double>& beta_k,
				/* [in] gamma_k */ x_line<double>& gamma_k )
{
	if (k < 0 || k >= probl.n)
	{
		TRACE ("no such pattern in set: %d / %d", k, probl.n - 1);
		exit (1);
	}

	// check for SV
	for (int i = 0 ; i < s_vect.len () ; i ++)
		if (s_vect[i].idx == k)
		{
			pattern_unlearn (true, i, eta_k, beta_k, gamma_k);
			return;
		}
 

	// check for RV
	for (int i = 0 ; i < r_vect.len () ; i ++)
		if (r_vect[i].idx == k)
		{
			pattern_unlearn (false, i, eta_k, beta_k, gamma_k);
			return;
		}

	INFO (0, "--> %d - vector not learned yet\n", k); 
}


void x_svm_r::calculate_etaC ( /* [out] result eta */ x_line<double>& eta )
{
	eta.ensure (s_vect.len () + 1);

	eta[0] = 0;
	for (int i = 0 ; i < r_vect.len () ; i ++)
		if (r_vect[i].type == x_elem::BOUNDED)
		{
			double b_k = (r_vect[i].lambda > 0 ? 1.0 : -1.0);
			eta[0] += b_k;
		}

	
	for (int i = 0 ; i < s_vect.len () ; i ++)
	{
		int s = s_vect[i].idx;
		eta[i + 1] = 0;
		for (int j = 0 ; j < r_vect.len () ; j ++)
			if (r_vect[j].type == x_elem::BOUNDED)
			{
				int k = r_vect[j].idx;
				double b_k = (r_vect[j].lambda > 0 ? 1.0 : -1.0);
				eta[i + 1] += (this->*get_q) (s, k) * b_k;
			}
	}
}


void x_svm_r::calculate_gammaC (/* [in] beta */ x_line<double>& beta,
				/* [out] gamma_c */ x_line<double>& gamma )
{
	int n = s_vect.len ();
	int m = r_vect.len ();
	gamma.ensure (m);

	for (int i = 0 ; i < m ; i ++)
	{
		int r = r_vect[i].idx;
		gamma[i] = beta[0];
		for (int j = 1 ; j < n + 1 ; j ++)
		{
			int s = s_vect[j - 1].idx;
			gamma[i] += (this->*get_q) (r, s) * beta[j];
		}
		for (int j = 0 ; j < m ; j ++)
			if (r_vect[j].type == x_elem::BOUNDED)
			{
				int k = r_vect[j].idx;
				double b_k = (r_vect[j].lambda > 0 ? 1.0 : -1.0);
				gamma[i] += (this->*get_q) (r, k) * b_k;
			}
	}
}


void x_svm_r::bound_first_unbounded (	/* in */ x_line<double>& beta_c,
					/* out */ int& sm, int& sp, double& delta_lm, double& delta_lp,
					/* inout */ x_line<int>& dup_m, x_line<int>& dup_p )
{
	// first case: determine how lambda_s are going to change and
	// determine the first reach to zero (sm) or to new C (sp)
	sm = sp = -1;
	delta_lm = delta_lp = 0;

	for (int i = 1 ; i < beta_c.len () ; i ++)
	{
		int iminus = i - 1;
		x_elem& s = s_vect[iminus];
		double h = calc_h (s.idx);
		double sh = h < 0 ? -1 : 1;
		if (sh < 0 && beta_c[i] <= 0 || sh > 0 && beta_c[i] >= 0)
		{
			if (s.lambda == 0)
				continue;

			double delta = (- sh * param.C - s.lambda) / (beta_c[i] + sh);

			INFO (4, "    %d[%g] sp: lambda: %g beta_c: %g delta: %g\n",
				s.idx, probl.y[s.idx], s.lambda, beta_c[i], delta);

			if (sp < 0)
			{
				// no vector yet
				sp = iminus;
				delta_lp = delta;
			} else if (IS_ZERO (delta - delta_lp)) {
				// duplicate delta found
				if (dup_p.len () == 0) {
					dup_p.add (sp);
					dup_p.add (iminus);
				} else if (dup_p[0] == sp) {
					dup_p.add (iminus);
				} else {
					while (dup_p.len () > 0)
						dup_p.remove (dup_p.len () - 1);
					dup_p.add (sp);
					dup_p.add (iminus);
				}
			} else if (delta > delta_lp) {
				sp = iminus;
				delta_lp = delta;
			}
		} else if (sh < 0 && beta_c[i] > 0 && beta_c[i] < 1 || sh > 0 && beta_c[i] > -1 && beta_c[i] < 0) {
			double delta_p = (- sh * param.C - s.lambda) / (beta_c[i] + sh);
			double delta_m = - s.lambda / beta_c[i];

			INFO (4, "    %d[%g] sm_sp: lambda: %g beta_c: %g delta_m: %g delta_p: %g\n",
				s.idx, probl.y[s.idx], s.lambda, beta_c[i], delta_m, delta_p);

			if (sm < 0)
			{
				// no vector yet
				sm = iminus;
				delta_lm = delta_m;
			} else if (IS_ZERO (delta_m - delta_lm)) {
				// duplicate delta found
				if (dup_m.len () == 0) {
					dup_m.add (sm);
					dup_m.add (iminus);
				} else if (dup_m[0] == sm) {
					dup_m.add (iminus);
				} else {
					while (dup_m.len () > 0)
						dup_m.remove (dup_m.len () - 1);
					dup_m.add (sm);
					dup_m.add (iminus);
				}
			} else if (delta_m > delta_lm) {
				sm = iminus;
				delta_lm = delta_m;
			}

			if (sp < 0)
			{
				// no vector yet
				sp = iminus;
				delta_lp = delta_p;
			} else if (IS_ZERO (delta_p - delta_lp)) {
				// duplicate delta found
				if (dup_p.len () == 0) {
					dup_p.add (sp);
					dup_p.add (iminus);
				} else if (dup_p[0] == sp) {
					dup_p.add (iminus);
				} else {
					while (dup_p.len () > 0)
						dup_p.remove (dup_p.len () - 1);
					dup_p.add (sp);
					dup_p.add (iminus);
				}
			} else if (delta_p > delta_lp) {
				sp = iminus;
				delta_lp = delta_p;
			}
		} else { // beta_c[i] >= 1
			double delta = - s.lambda / beta_c[i];
			INFO (4, "    %d[%g] sm: lambda: %g beta_c: %g delta: %g\n",
				s.idx, probl.y[s.idx], s.lambda, beta_c[i], delta);

			if (s.lambda == -param.C || s.lambda == param.C)
				continue;

			if (sm < 0)
			{
				// no vector yet
				sm = iminus;
				delta_lm = delta;
			} else if (IS_ZERO (delta - delta_lm)) {
				// duplicate delta found
				if (dup_m.len () == 0) {
					dup_m.add (sm);
					dup_m.add (iminus);
				} else if (dup_m[0] == sm) {
					dup_m.add (iminus);
				} else {
					while (dup_m.len () > 0)
						dup_m.remove (dup_m.len () - 1);
					dup_m.add (sm);
					dup_m.add (iminus);
				}
			} else if (delta > delta_lm) {
				sm = iminus;
				delta_lm = delta;
			}
		}
	}
}


void x_svm_r::bound_second_rest (	/* in */ x_line<double>& gamma_c,
				/* out */ int& ro, int& re, double& delta_lo, double& delta_le,
				/* inout */ x_line<int>& dup_o, x_line<int>& dup_e )
{
	// second: determine how h_r are going to change and
	// determine first reach to zero from positive (ro) or negative (re) h_r's
	ro = re = -1;
	delta_lo = delta_le = 0;

	for (int i = 0 ; i < gamma_c.len () ; i ++)
	{
		x_elem& r = r_vect[i];
		double h = calc_h (r.idx);
		double sg = (gamma_c[i] < 0 ? -1.0: 1.0);
		double sl = (r.lambda < 0 ? -1.0 : 1.0);

		if (r.type == x_elem::ZERO)
		{
			double delta = ((- sg) * eps - h) / gamma_c[i];
			INFO (4, "    %d[%g] lo: lambda: %g h_k: %g gamma_c: %g delta: %g\n",
				r.idx, probl.y[r.idx], r.lambda, h, gamma_c[i], delta);

			if (ro < 0) {
				ro = i;
				delta_lo = delta;
			} else if (IS_ZERO (delta - delta_lo)) {
				// duplicate delta found (duplicate patterns)
				if (dup_o.len () == 0)
				{
					dup_o.add (ro);
					dup_o.add (i);
				} else if (dup_o[0] == ro) {
					dup_o.add (i);
				} else {
					while (dup_o.len () > 0)
						dup_o.remove (dup_o.len () - 1);
					dup_o.add (ro);
					dup_o.add (i);
				}
			} else if (delta > delta_lo) {
				ro = i;
				delta_lo = delta;
			}
		} else { // bounded, r_vect[iminus].type == x_elem::BOUNDED
			if (sg != sl)
			{
				double delta = (sg * eps - h) / gamma_c[i];
				INFO (4, "    %d[%g] le: lambda: %g h_k: %g gamma_c: %g delta: %g\n",
					r.idx, probl.y[r.idx], r.lambda, h, gamma_c[i], delta);


				if (re < 0)
				{
					re = i;
					delta_le = delta;
				} else if (IS_ZERO (delta - delta_le)) {
					// duplicate delta found (duplicate pattern)
					if (dup_e.len () == 0)
					{
						dup_e.add (re);
						dup_e.add (i);
					} else if (dup_e[0] == re) {
						dup_e.add (i);
					} else {
						while (dup_e.len () > 0)
							dup_e.remove (dup_e.len () - 1);
						dup_e.add (re);
						dup_e.add (i);
					}
				} else if (delta > delta_le) {
					re = i;
					delta_le = delta;
				}
			}
		}
	}
}


void x_svm_r::decrease_c (int incr)
{
	// test whether we have error vectors
// 	{
// 		bool exist = false;
// 		for (int i = 0 ; i < r_vect.len () ; i ++)
// 			if (r_vect[i].type == x_elem::BOUNDED)
// 			{
// 				exist = true;
// 				break;
// 			}
// 		if (!exist)
// 		{
// 			INFO (0, "--> configuration does not have error vectors (E)");
// 			return;
// 		}
// 	}

	x_line<double> eta_c;
	x_line<double> beta_c;
	x_line<double> gamma_c;

	int cycle = incr;
	do {
		INFO (1, "===[ decreasing C: %g - cycle %d ]===========================\n", param.C, cycle);
		if (g_dbgLevel > 2)
			print_state ();

		calculate_etaC (eta_c);
		calculate_beta (eta_c, beta_c);
		calculate_gammaC (beta_c, gamma_c);

		// maximas
		double delta_l[5];
		bool enable[5];

		// store positions of duplicate minimas (found for (O) to S migration)
		x_line<int> dup[4];
		
		// determine the smallest negative increase of dC before dl_s reaches limits
		// first case: determine which vectors from S are first to reaching limits
		int sm = -1, sp = -1;
		bound_first_unbounded (beta_c, sm, sp, /* lm */ delta_l[0], /* lp */ delta_l[1], dup[0], dup[1]);
		enable[0] = sm < 0 ? false : true;
		enable[1] = sp < 0 ? false : true;
		INFO (2, "delta_lm: %g (%d) ->%s\n", delta_l[0], sm < 0 ? -1 : s_vect[sm].idx, enable[0] ? "true" : "false");
		INFO (2, "delta_lp: %g (%d) ->%s\n", delta_l[1], sp < 0 ? -1 : s_vect[sp].idx, enable[1] ? "true" : "false");

		// second case: determine which gradients corresponding to vectors from R will first reach zero
		int ro = -1, re = -1;
		bound_second_rest (gamma_c, ro, re, /* lo */ delta_l[2], /* le */ delta_l[3], dup[2], dup[3]);
		enable[2] = ro < 0 ? false : true;
		enable[3] = re < 0 ? false : true;
		INFO (2, "delta_lo: %g (%d) ->%s\n", delta_l[2], ro < 0 ? -1 : r_vect[ro].idx, enable[2] ? "true" : "false");
		INFO (2, "delta_le: %g (%d) ->%s\n", delta_l[3], re < 0 ? -1 : r_vect[re].idx, enable[3] ? "true" : "false");

		// fourth case: lambda_c should not underflow an imposed limit
		enable[4] = true;
		delta_l[4] = param.newC - param.C;
		INFO (2, "delta_lc: %g ->%s\n", delta_l[4], enable[4] ? "true" : "false");

		// find the maximum from the five elements
		int max_idx = -1;
		find_max_delta (enable, delta_l, 5, max_idx);
		if (delta_l[max_idx] > 0)
		{
			TRACE ("positive delta found: %g\n", delta_l[max_idx]);
			exit (1);	
		}

		// update w0 and lambda_s accordingly
		{
			double dlc = delta_l[max_idx];
			INFO (2, "dlc: %g\n", dlc);

			param.C += dlc;
			update_margins (dlc, beta_c);

			for (int i = 0 ; i < r_vect.len () ; i ++)
				if (r_vect[i].type == x_elem::BOUNDED)
				{
					double sl = r_vect[i].lambda < 0 ? -1.0 : 1.0;
					r_vect[i].lambda = sl * param.C;
				}
		}

		// determine which vector is going to migrate between sets
		switch (max_idx)
		{
		case 0:
			// k in S, sm
			// take out vector sm from S and put it to R (O)
			INFO (2, "--> %d (S) migrates to O\n", s_vect[sm].idx); 
			{
				int idx_sm = s_vect[sm].idx;
				if (!IS_ZERO2 (s_vect[sm].lambda))
				{
					TRACE ("lambda_sm: %g should be zero!\n", s_vect[sm].lambda);
					exit (1);
				}
				x_elem l = {idx_sm, 0, x_elem::ZERO};
				s_vect.remove (sm);
				r_vect.add (l);

				// update P and Q
				contract_pq (sm);

				if (dup[0].len () && sm == dup[0][0])
				{
					for (int i = 1 ; i < dup[0].len () ; i ++)
					{
						// one vector already removed from S
						int id = dup[0][i] - i;
						INFO (2, "--> duplicate result found for sm: %d\n", idx_sm);

						INFO (2, "--> %d (S) migrates to O\n", s_vect[id].idx);
						if (!IS_ZERO2 (s_vect[id].lambda))
						{
							TRACE ("lambda_smid: %g should be zero!\n", s_vect[id].lambda);
							exit (1);
						}
						x_elem l = {s_vect[id].idx, 0, x_elem::ZERO};
						s_vect.remove (id);
						r_vect.add (l);
		
						// update P and Q
						contract_pq (id);
					}
				}
			}
			break;
		case 1:
			// k in S, sp
			// take out vector sp from S and put it to R (E)
			INFO (2, "--> %d (S) migrates to E\n", s_vect[sp].idx);
			{
				int idx_sp = s_vect[sp].idx;
				if (!IS_CONST2(s_vect[sp].lambda, param.C) && !IS_CONST2(s_vect[sp].lambda, -param.C))
				{
					TRACE ("lambda_sp: %g should be %g!\n", s_vect[sp].lambda, param.C);
					exit (1);
				}
				x_elem l = {idx_sp, (s_vect[sp].lambda < 0 ? -1 : 1) * param.C, x_elem::BOUNDED};
				s_vect.remove (sp);
				r_vect.add (l);

				// update P and Q
				contract_pq (sp);

				if (dup[1].len () && sp == dup[1][0])
				{
					for (int i = 1 ; i < dup[1].len () ; i ++)
					{
						// one vector already removed from S
						int id = dup[1][i] - i;
						INFO (2, "--> duplicate result found for sp: %d\n", idx_sp);

						INFO (2, "--> %d (S) migrates to E\n", s_vect[id].idx);
						if (	!IS_CONST2(s_vect[id].lambda, param.C) && 
							!IS_CONST2(s_vect[id].lambda, -param.C) )
						{
							TRACE ("lambda_spid: %g should be +/-%g!\n",
								s_vect[id].lambda, param.C);
							exit (1);
						}
						x_elem l = {s_vect[id].idx, (s_vect[id].lambda < 0 ? -1 : 1) * param.C,
							x_elem::BOUNDED};
						s_vect.remove (id);
						r_vect.add (l);
		
						// update P and Q
						contract_pq (id);
					}
				}
			}
			break;
		case 2:
			// k in R (O), ro
			// we have a support vector candidate, transferred from O to S
			{
				int idx_ro = r_vect[ro].idx;
				double h = calc_h (idx_ro);
				INFO (2, "--> %d (O) migrates to S\n", r_vect[ro].idx);
				if (!IS_CONST2 (h, eps) && !IS_CONST2 (h, -eps))
				{
					TRACE ("gradient h_ro: %g should be eps!\n", h);
					exit (1);
				}
				if (!IS_ZERO2 (r_vect[ro].lambda))
				{
					TRACE ("lambda_ro: %g should be zero!\n", r_vect[ro].lambda);
					exit (1);
				}
				x_elem l = {r_vect[ro].idx, 0, x_elem::UNBOUNDED};
				r_vect.remove (ro);
				s_vect.add (l);

				// update P and Q
				expand_pq (l.idx);
				if (dup[2].len () && ro == dup[2][0])
				{
					for (int i = 1 ; i < dup[2].len () ; i ++)
					{
						// we already removed some vector(s)
						int id = dup[2][i] - i;
						INFO (2, "--> duplicate result found for ro: %d\n", idx_ro);

						double h = calc_h (r_vect[id].idx);
						INFO (2, "--> %d (O) migrates to S\n", r_vect[id].idx);
						if (!IS_CONST2 (h, eps) && !IS_CONST2 (h, -eps))
						{
							TRACE ("gradient h_ro(%d): %g should be +/- %g!\n",
								r_vect[id].idx, h, eps);
							exit (1);
						}
						if (!IS_ZERO2 (r_vect[id].lambda))
						{
							TRACE ("lambda_ro(%d): %g should be zero!\n",
								r_vect[id].idx, r_vect[id].lambda);
							exit (1);
						}
						x_elem l = {r_vect[id].idx, 0, x_elem::UNBOUNDED};
						r_vect.remove (id);
						s_vect.add (l);
		
						// update P and Q
						expand_pq (l.idx);
					}
				}
			}
			break;
		case 3:
			// k in R (E), re
			// we have a support vector candidate, transferred from E to S
			{
				int idx_re = r_vect[re].idx;
				double h = calc_h (idx_re);

				INFO (2, "--> %d (E) migrates to S\n", idx_re);
				if (!IS_CONST2 (h, eps) && !IS_CONST2 (h, -eps))
				{
					TRACE ("gradient h_re: %g should be eps!\n", h);
					exit (1);
				}
				if (!IS_CONST2 (r_vect[re].lambda, param.C) && !IS_CONST2 (r_vect[re].lambda, -param.C))
				{
					TRACE ("lambda_re: %g should be %g!\n", r_vect[re].lambda, param.C);
					exit (1);
				}

				x_elem l = {idx_re, (r_vect[re].lambda < 0 ? -1 : 1) * param.C, x_elem::UNBOUNDED};
				r_vect.remove (re);
				s_vect.add (l);

				// update P and Q
				expand_pq (l.idx);

				if (dup[3].len () && re == dup[3][0])
				{
					for (int i = 1 ; i < dup[3].len () ; i++)
					{
						// we already removed some vector(s)
						int id = dup[3][i] - i;
						INFO (2, "--> duplicate result found for re: %d\n", idx_re);

						double h = calc_h (r_vect[id].idx);
						INFO (2, "--> %d (E) migrates to S\n", r_vect[id].idx); 
						if (!IS_CONST2 (h, eps) && !IS_CONST2 (h, -eps))
						{
							TRACE ("gradient h_re(%d): %g should be +/-%g!\n",
								r_vect[id].idx, h, eps);
							exit (1);
						}
						if (	!IS_CONST2 (r_vect[id].lambda, param.C) && 
							!IS_CONST2 (r_vect[id].lambda, -param.C) )
						{
							TRACE ("lambda_re(%d): %g should be +/-%g!\n",
								r_vect[id].idx, r_vect[id].lambda, param.C);
							exit (1);
						}
						x_elem l = {r_vect[id].idx, (r_vect[id].lambda < 0 ? -1 : 1) * param.C,
							x_elem::UNBOUNDED};
						r_vect.remove (id);
						s_vect.add (l);
	
						// update P and Q
						expand_pq (l.idx);
					}
				}
			}
			break;
		case 4:
			// all deltas are greater than maximum increment before reaching C
			break;
		}

		if (param.saveEvery > 0 && (cycle % param.saveEvery) == 0 && cycle > 0)
		{
			char file_name [100];
			char prefix [100];
			for (unsigned i = 0 ; i < strlen (param.saveTo) ; i ++)
			{
				prefix[i] = param.saveTo[i];
				if (prefix[i] == '-')
				{
					prefix[i] = 0;
					break;
				}
			}
			sprintf (file_name, "c-state-%s-%d.idsvm", prefix, cycle);
			INFO (1, "===[ saving state %s ... ]=======================================\n", file_name);
			save_state (perm, file_name);
			restore_state (perm, file_name);
			INFO (1, "===[ state %s saved ]==========================================\n", file_name);
		}

		cycle ++;
	} while (param.C > param.newC);

	{
		cycle --;
		char file_name [100];
		char prefix [100];
		for (unsigned i = 0 ; i < strlen (param.saveTo) ; i ++)
		{
			prefix[i] = param.saveTo[i];
			if (prefix[i] == '-')
			{
				prefix[i] = 0;
				break;
			}
		}
		sprintf (file_name, "c-state-%s-%d.idsvm", prefix, cycle);
		INFO (1, "===[ saving state %s ... ]=======================================\n", file_name);
		save_state (perm, file_name);
		restore_state (perm, file_name);
		INFO (1, "===[ state %s saved ]==========================================\n", file_name);
	}
}


void x_svm_r::train (void)
{
	int incr = 0;

	if (!param.saveTo && !(param.info & PARAM_VIEW_STATE))
	{
		TRACE ("-w parameter missing");
		exit (1);
	}
	if (param.saveTo)
	{
		unsigned i = 0;
		for (i = 0 ; i < strlen (param.saveTo) ; i ++)
			if (param.saveTo[i] == '-')
				break;
		if (i >= strlen (param.saveTo))
		{
			TRACE ("invalid format of -w parameter");
			exit (1);
		}
	}

	if (param.info & PARAM_PRECALCULATE)
		precalculate ();

	// check for restoration needed
	if (param.restoreFrom)
	{
		unsigned i = 0;
		for (i = 0 ; i < strlen (param.restoreFrom) ; i ++)
			if (param.restoreFrom[i] == '-')
				break;
		i ++;
		incr = atoi (param.restoreFrom + i);
		if (incr < 0)
		{
			TRACE ("invalid name given: %s", param.restoreFrom);
			exit (1);
		}
		if (param.info & PARAM_RESTORE_C)
		{
			char file_name [100];
			sprintf (file_name, "c-rstate-%s.idsvm", param.restoreFrom);
			INFO (1, "===[ restoring c-state %s ... ]=======================================\n", file_name);
			restore_state (perm, file_name);
			INFO (1, "===[ c-state %s restored ]==========================================\n", file_name);
			incr ++;
		} else  {
			char file_name [100];
			sprintf (file_name, "rstate-%s.idsvm", param.restoreFrom);
			INFO (1, "===[ restoring state %s ... ]=======================================\n", file_name);
			restore_state (perm, file_name);
			INFO (1, "===[ state %s restored ]==========================================\n", file_name);
			incr ++;
		}
	} else {
		// alloc permutation of input vectors
		perm.ensure (probl.n);

		// generate permutation
		for (int i = 0 ; i < probl.n ; i ++)
			perm [i] = i;
		for (int i = 0 ; i < probl.n - 1 ; i ++)
		{
			int j = i + rand () % (probl.n - i);
			swap (perm[i], perm[j]);
		}

		// [debug] restore/save perm
		if (param.info & PARAM_BUG_FIND)
		{
			FILE * f = fopen ("perm", "rb");
			perm.restore (f);
			INFO (4, "*** perm restored ***\n");
			fclose (f);
		} else {
			FILE * f = fopen ("perm", "wb");
			perm.store (f);
			fclose (f);
		}
	}

super_ugly_loop:
	// restoration was made from state 0 or we just begun
	if (incr <= 1)
	{
		// assure that the first two vectors determine a negative initial lambda
		int i = 1;
		double lambda = 0;
		do {
			// calculate lambda
			int u = perm[0];
			int v = perm[i];
			double q12 = (this->*get_q) (u, v);
			double q22 = (this->*get_q) (v, v);
			double q11 = (this->*get_q) (u, u);
			lambda = (probl.y[perm[0]] - probl.y[perm[i]] + 2 * eps) / (q11 - q12 - q12 + q22);
			w0 = (probl.y[perm[0]] + probl.y[perm[i]]) - (q11 - q22) * lambda;
			w0 /= 2.00;

			if (lambda > 0)
				i ++;
		} while (lambda > 0 && i < perm.length);
		if (i >= perm.length)
		{
			TRACE ("invalid perm, negative lambda not found (least probable)\n");
			exit (1);
		}
		swap (perm[1], perm[i]);
		
		// adjust C to nearest value
		{
			double c = (int((abs(lambda) + 1) * 10)) / 10.0;
			if (c > param.C)
				param.C = c;
		}
		INFO (2, "--> initial lambda: %g w0: %g C: %g\n", lambda, w0, param.C);
	
		// add these vectors as margin support vectors
		incr = 0;
		s_vect.ensure (0);
		r_vect.ensure (0);
		p.ensure (0);
		q.ensure (0);
		for (int i = 0 ; i < 2 ; i ++)
		{
			x_elem l = {perm[i], (i == 0) ? lambda : - lambda, x_elem::UNBOUNDED};
			s_vect.add (l);

			if (incr == 0)
				create_pq (perm[incr]);
			else
				expand_pq (perm[incr]);
			
			incr ++;
		}

		// save state 0
		{
			char file_name [100];
			char prefix [100];
			for (unsigned i = 0 ; i < strlen (param.saveTo) ; i ++)
			{
				prefix[i] = param.saveTo[i];
				if (prefix[i] == '-')
				{
					prefix[i] = 0;
					break;
				}
			}

			sprintf (file_name, "rstate-%s-%d.idsvm", prefix, 0);
			INFO (1, "===[ saving state %s ... ]=======================================\n", file_name);
			save_state (perm, file_name);
			restore_state (perm, file_name);
			INFO (1, "===[ state %s saved ]==========================================\n", file_name);
		}
	}

	if (param.test_file)
	{
		read_test ();
		evaluate ();
		exit (1);
	}
	if (param.info & PARAM_DECREASE_C)
	{
		// decrease cost C
		decrease_c (incr);
		print_state ();
		exit (1);
	}
	INFO (1, "--> initial C: %g\n", param.C);
	if (param.info & PARAM_VIEW_STATE)
	{
		// print context info
		print_state ();
		exit (1);
	}

	// main loop, incrementally learn patterns
	x_line<double> eta_k;
	x_line<double> beta_k;
	x_line<double> gamma_k;

	while (incr < probl.n)
	{
		// display support vectors' coefficients
		// print_state ();
		// s_vect.print ("SVs");

		// learn pattern
		int u = perm[incr];
		int res = pattern_learn (incr, u, eta_k, beta_k, gamma_k);
		if (res == -1)
		{
			incr = 0;
			int first = perm[0];
			for (int i = 0 ; i < probl.n - 2 ; i ++)
				perm[i] = perm[i + 1];
			perm[probl.n - 1] = first;

			goto super_ugly_loop;
		}

		if (param.saveEvery > 0 && (incr % param.saveEvery) == 0 && incr > 0)
		{
			char file_name [100];
			char prefix [100];
			for (unsigned i = 0 ; i < strlen (param.saveTo) ; i ++)
			{
				prefix[i] = param.saveTo[i];
				if (prefix[i] == '-')
				{
					prefix[i] = 0;
					break;
				}
			}
			sprintf (file_name, "rstate-%s-%d.idsvm", prefix, incr);
			INFO (1, "===[ saving state %s ... ]=======================================\n", file_name);
			save_state (perm, file_name);
			restore_state (perm, file_name);
			INFO (1, "===[ state %s saved ]==========================================\n", file_name);
		}

		// go to the next pattern
		incr ++;
	}

	{
		incr --;
		char file_name [100];
		char prefix [100];
		for (unsigned i = 0 ; i < strlen (param.saveTo) ; i ++)
		{
			prefix[i] = param.saveTo[i];
			if (prefix[i] == '-')
			{
				prefix[i] = 0;
				break;
			}
		}
		sprintf (file_name, "rstate-%s-%d.idsvm", prefix, incr);
		INFO (1, "===[ saving state %s ... ]=======================================\n", file_name);
		save_state (perm, file_name);
		restore_state (perm, file_name);
		INFO (1, "===[ state %s saved ]==========================================\n", file_name);
	}
}


double x_svm_r::calc_g (x_svm_node * x)
{
	volatile double sum = 0;

	// contribution of support vectors
	for (int i = 0 ; i < s_vect.len () ; i ++)
	{
		int idx = s_vect[i].idx;
		x_svm_node * s = probl.x[idx];
		sum += pkern->K_ext (s, x) * s_vect[i].lambda;
	}

	// contribution of error vectors
	for (int i = 0 ; i < r_vect.len () ; i ++)
		if (r_vect[i].type == x_elem::BOUNDED)
		{
			int idx = r_vect[i].idx;
			x_svm_node * s = probl.x[idx];
			sum += pkern->K_ext (s, x) * r_vect[i].lambda;
		}
	
	sum += w0;
	return sum;
}


void x_svm_r::evaluate (void)
{
	double mse = 0, mae = 0;
	int p = 0, n = 0;
	printf ("C: %g sigma: %g\n", param.C, param.sigma);
	printf ("Support Vectors: %d\n", s_vect.len ());
	for (int i = 0 ; i < s_vect.len () ; i ++)
	{
		int s = s_vect[i].idx;
		x_svm_node * v = probl.x[s];
		double g = calc_g (v);
		double h = g - probl.y[s];
		printf ("  %d[%g]: %lg\n", s, probl.y[s], h);

		bool miss = !IS_CONST2(h, eps) && !IS_CONST2(h, -eps);
		if (miss)
		{
			TRACE ("support vector: %d missclassified, h: %.14lf!\n", s, h);
			//exit (1);
		}

		mse += h * h;
		mae += abs (h);
		n ++;
	}

	printf ("Rest Vectors: %d\n", r_vect.len ());
	for (int i = 0 ; i < r_vect.len () ; i ++)
	{
		int r = r_vect[i].idx;
		x_svm_node * v = probl.x[r];
		double g = calc_g (v);
		double h = g - probl.y[r];
		printf (" %d[%g]: %lg\n", r, probl.y[r], h);

		if (h <= eps && h >= -eps)
			p ++;

		mse += h * h;
		mae += abs (h);
		n ++;
	}
	printf ("learning accuracy: %d/%d (%3.2f%%)\n", p + s_vect.len (), r_vect.len () + s_vect.len (),
		(double)(p + s_vect.len ()) * 100.0 / (r_vect.len () + s_vect.len ())
	);
	printf ("train MSE: %g MAE: %g\n", mse / n, mae / n);

	p = n = 0;
	mae = mse = 0;
	printf ("Test Vectors: %d\n", test.n);
	for (int i = 0 ; i < test.n ; i ++)
	{
		x_svm_node * v = test.x[i];
		double g = calc_g (v);
		double h = g - test.y[i];
		printf (" %d[%g]: %lg\n", i, test.y[i], h);

		mse += h * h;
		mae += abs (h);
		n ++;
	}
	printf ("test MSE: %g MAE: %g\n", mse / n, mae / n);
}
