% notice, this leaveOneTrain is restricted to PosOnly


cross_validation(AllEIs,PredictiveAccurracy):-
	get_folds(AllEIs,AllFolds),print_list(AllFolds),nl,
	maplist(train_and_test(AllEIs),AllFolds,Results),
	%nl,write('********Prediction on each fold********),nl,
	writeNumInColumn(Results),
	average(Results,PredictiveAccurracy).


get_folds(Fold_Index,AllEIs,OneFold):-
	set(cross_validation_method,leave_one_out),!,
	nth(Fold_Index,AllEIs,FoldEI),
	wrapEle_As_List(FoldEI,OneFold).
	
get_folds(Fold_Index,AllEIs,OneFold):-
	fold(Fold_Index,OneFold,AllEIs).
	% fold0(1,FoldEIs) -- please see cancer's foldInfo.

/*get_folds(AllEIs,AllFolds):-
	set(cross_validation_method,leave_one_out),!,
	maplist(wrapEle_As_List,AllEIs,AllFolds).	
get_folds(AllEIs,AllFolds):-
	findall(FoldEIs,fold(FoldIndex,FoldEIs,AllEIs),AllFolds).
*/

wrapEle_As_List(X,[X]).

% if it is already predictable by default, then there is no need to derive the hypothesis
train_and_test(AllEIs,FoldEIs,100.0):-	
	maplist(default_predict,FoldEIs),!,
	nl,write('--------------------Test'),
	write({FoldEIs,all_predicted_by_default,100.0}),nl,nl,nl.

train_and_test(AllEIs,FoldEIs,Result):-	
	write('checkPredictedByDefault'),test(FoldEIs,Result),Result=100.0,!.

train_and_test(AllEIs,FoldEIs,Result):-	
	retractall(candidateH0(EI,TI,OneH)),
	retractall(connect(OneNode,AnotherNode,Flux)),
	%retractall(interpreted(ClaID,Cla)), -- it will make the whole process a bit quicker
	
		(set(etoB,yes)->
			maplist(removeTestEI,FoldEIs,RemovedEIsDetail); 
			Flag=0	% Flag is for later added
		),

	%nl,write('-------------------start one iteration --------------'),nl,
	(set(default_evaluation,yes) -> % compute the default accuracies
		Hypothesis=[];
		TestEIs=FoldEIs, ord_subtract(AllEIs,TestEIs,TrainingEIs),	
		train(TrainingEIs,Hypothesis)
	),

% put it backk before testing, it is because I re-use unCoverRecord for testing -- it will take out the one to be covered. when the testEIs is more than one (e.g. cancer 10-fold), then you need to put all the test data back
% handle a list of test EI (it is not simply maplist, because you only record those uncover); get ev Test. 
	(Flag==0 ->	% the reason of not simply calling set(etoB,yes) again is to handel the secondary example -- see covering.pl
		Foo2=1;	% do nothing
		maplist(addTestEI,RemovedEIsDetail)
	),

	addT(Hypothesis,Ref),
	t(1),
	%listing,
	forAll(closedWorldAssumption(Cla),(assertz(Cla,ClaRef),assertz(assertedCla(ClaRef)))),

	test(FoldEIs,Result),

	t(2),
	forAll(assertedCla(AnyAssertedClaRef),(erase(AnyAssertedClaRef),retractall(assertedCla(AnyAssertedClaRef)))), % this is for deleting version
	removeT(Ref),
	write('-------------------Finish one left-out-----'),write({FoldEIs,Result}),nl. %,listing.


removeTestEI(EI,{EI,E,PosNegSign}):-
	retract(ex(EI,E,PosNegSign)).

addTestEI({EI,E,PosNegSign}):-
	asserta(ex(EI,E,PosNegSign)).




/********************************** Training ****************************************/
train(TrainingEIs,Hypothesis):-
	sepPN_ID(TrainingEIs,TrainPE,TrainNE),
	learn(TrainPE,TrainNE,Hypothesis).


/********************************** Test ****************************************/
test(EIs,Result):-
	%nl,write('--------------------Test'),
	maplist(predictTestData,EIs,CorrectPredictedEIs0), % the reason for using maplist, rather than directly unCoverRecord is to support default within the TestEIs
	flatten(CorrectPredictedEIs0,CorrectPredictedEIs),
	percentage(CorrectPredictedEIs,EIs,Result). % Result is a number between 1 and 100, since you scale it with 100 when using the predicate percentage
	%write({EIs,CorrectPredictedEIs,Result}),nl,nl,nl.

percentage(Record,All,P):-
	length(Record,K),
	length(All,AllK),
	P is (K*100)/AllK.




% considering negative examples, you can't simply uncover(WholeList) -- then further refine out true or false.


predictTestData(TestID,[TestID]):-
		set(ignore_predictDefault,yes),
		default_predict(TestID),!.

predictTestData(TestID,Result):-
		unCoverRecord([TestID],CoverResult), % need to deal with evaluation Depth and e to b problem 
		ex(TestID,E,PosNegSign),
		trueCheck(CoverResult,PosNegSign,CorrectSign),
		(CorrectSign=[1] ->
			Result=[TestID];
			Result=[]
		).


trueCheck([],1,[1]).
trueCheck([E],0,[1]).
trueCheck([],0,[]).
trueCheck([E],1,[]).
% TruePositive, FalsePos,
% TrueNeg, FalseNeg





lv_readFile:-
	yap_flag(argv,[InputFile]),
  	consult(InputFile), 
	%tell('grammarLeaveOneOut.txt'),	
	findall(EIp,ex(EIp,Ep,1),PosEI0s), % full size learning
	findall(EIn,ex(EIn,En,0),NegEIs),
	length(NegEIs,NK),

	unCoverRecord(PosEI0s,PosEIs), % reduce = remove those already explained by the examples
	length(PosEIs,PK),
	append(PosEIs,NegEIs,AllEI),
	write({PK,NK}),write('Totally '),nl,
	leaveOneOut(AllEI,AllEI,Results),
	
	% Results is a list of Uncovered test data -- you may trace the learning result and see why
	percentage(Results,AllEI,PA),
	write('Predictive Accuracy is'),write(PA),nl,
	nl.%told.