import os, subprocess, sys, getopt
import utils, gdv_utils
from LatexBuilder import LatexBuilder
import pr_roc_utils, pca_utils, draw_utils, ground_truth_utils
import os.path
import settings
import numpy as np
import pandas as pd
import math


gnuplotLinesTemplate = file(settings.GNUPLOT_LINES_TEMPLATE_FILE, 'r').read()
gnuplotHeatmapTemplate = file(settings.GNUPLOT_HEATMAP_TEMPLATE_FILE, 'r').read()

def getLinePlot(distances_dir, methods, x_label, y_label, output_suffix = ''):

    if not isinstance(methods,list):
        methods = [methods]
    if not isinstance(x_label,list):
        x_label = [x_label]
    if not isinstance(y_label,list):
        y_label = [y_label]
    if output_suffix != '':
        output_suffix = '_' + output_suffix

    plots_directory = distances_dir + 'plots/%s_%s/' % ('-'.join(x_label),'-'.join(y_label))


    if not os.path.exists(plots_directory):
        os.makedirs(plots_directory)
    if not os.path.exists(plots_directory + 'eps/'):
        os.makedirs(plots_directory + 'eps/')
    input_descriptions = []
    for m in methods:
        is_random_method = m.find('random') == 0
        #print 'method: %s' %m
        enrichment_file = '../../%s_pr_roc.txt' % m

        for x in x_label:
            for y in y_label:
                series_title = gdv_utils.getMethodName(m, onlyType=False) if not is_random_method else m.capitalize()
                #print 'series title: '+series_title
                #print 'xt: %s | yt: %s | series: %s ' %(x,y,series_title)
                if len(y_label)>1:
                    series_title = pr_roc_utils.get_label_title(y) + " - " + series_title
                if len(x_label)>1:
                    series_title = pr_roc_utils.get_label_title(x) + " - " + series_title

                if not is_random_method:
                    input_descriptions.append(
                        '"%s" using %s:%s title "%s" with lines' % (enrichment_file, pr_roc_utils.get_pr_roc_file_column_number(x), pr_roc_utils.get_pr_roc_file_column_number(y),series_title)
                    )
                else:
                    x_col = pr_roc_utils.get_pr_roc_file_column_number(x,contains_std_dev=True)
                    y_col = pr_roc_utils.get_pr_roc_file_column_number(y,contains_std_dev=True)

                    input_descriptions.append(
                                '"%s" using %s:%s title "%s" with lines' % (enrichment_file, x_col, y_col, series_title)
                        )

                    # if x_col != 1:
                    #     input_descriptions.append(
                    #                 '"%s" using %s:%s:%s:%s title "%s" with xyerrorbars' % (enrichment_file, x_col,x_col+1, y_col, y_col+1, series_title)
                    #         )
                    # else:
                    #     input_descriptions.append(
                    #                 '"%s" using %s:%s:%s title "%s" with yerrorbars' % (enrichment_file, x_col, y_col, y_col+1, series_title)
                    #         )


    figure_name = '%s_%s_%s%s' % (distance_measure,'-'.join(x_label),'-'.join(y_label), output_suffix)

    #set template values
    template_values = {
        'title': '',
        'x_label':  ','.join([pr_roc_utils.get_label_title(x) for x in x_label]),
        'y_label': ','.join([pr_roc_utils.get_label_title(y) for y in y_label]),
        'input_description': ', '.join(input_descriptions),
        'output_file': './eps/'  + figure_name,
    }

    gnuplotFile = figure_name + '.gnuplot'

    #write gnuplot file
    open(plots_directory + gnuplotFile, mode='wb').write(gnuplotLinesTemplate % template_values)

    #run gnuplot
    p = subprocess.Popen(['gnuplot', gnuplotFile], stdout=subprocess.PIPE, cwd = plots_directory)
    p_output = p.stdout.read()
    if p_output != '':
        print(p_output)

    return '%s_%s/' % ('-'.join(x_label),'-'.join(y_label)) + 'eps/' + figure_name + '.eps'

def getBarPlot(distances_dir, methods, x_label, y_label, input_file, input_file_methods,output_suffix = ''):
    gnuplotTemplate = file(settings.GNUPLOT_BAR_TEMPLATE_FILE, 'r').read()
    if not isinstance(methods,list):
        methods = [methods]
    if not isinstance(x_label,list):
        x_label = [x_label]
    if not isinstance(y_label,list):
        y_label = [y_label]
    if output_suffix != '':
        output_suffix = '_' + output_suffix

    plots_directory = distances_dir + 'plots/'
    plots_sub_directory = ('%s/' % ('-'.join(y_label))).lower()
    plots_directory += plots_sub_directory

    if not os.path.exists(plots_directory):
        os.makedirs(plots_directory)
    if not os.path.exists(plots_directory + 'eps/'):
        os.makedirs(plots_directory + 'eps/')
    input_descriptions = []
    for m in methods:
        is_random_method = m.find('random') == 0
        for x in x_label:
            for y in y_label:
                series_title = gdv_utils.getMethodName(m, onlyType=False) if not is_random_method else m.capitalize()
                if len(y_label)>1:
                    series_title = pr_roc_utils.get_label_title(y) + " - " + series_title
                if len(x_label)>1:
                    series_title = pr_roc_utils.get_label_title(x) + " - " + series_title

                if not is_random_method:
                    input_descriptions.append('\'%s\' using %s:%s:xtic(1) title "%s"' % (input_file, 2*input_file_methods.index(m)+1, 2*input_file_methods.index(m)+2, series_title))
                else:
                    input_descriptions.append('\'%s\' using %s:%s:xtic(1) title "%s"' % (input_file, 2*len(input_file_methods)+1, 2*len(input_file_methods)+2, series_title))
			
    figure_name =('%s_%s%s' % (distance_measure,'-'.join(y_label), output_suffix)).lower()

    #set template values
    template_values = {
        'title': '',
        'x_label':  ','.join([pr_roc_utils.get_label_title(x) for x in x_label]),
        'y_label': ','.join([pr_roc_utils.get_label_title(y) for y in y_label]),
        'input_description': ', '.join(input_descriptions),
        'output_file': './eps/'  + figure_name,
    }

    gnuplotFile = figure_name + '.gnuplot'

    #write gnuplot file
    open(plots_directory + gnuplotFile, mode='wb').write(gnuplotTemplate % template_values)

    #run gnuplot
    p = subprocess.Popen(['gnuplot', gnuplotFile], stdout=subprocess.PIPE, cwd = plots_directory)
    p_output = p.stdout.read()
    if p_output != '':
        print(p_output)

    return plots_sub_directory + 'eps/' + figure_name + '.eps'


def getHeatmap(distances_dir, methods, matrix_file):

    plots_directory = distances_dir + 'plots/intersections/'

    if not os.path.exists(plots_directory):
        os.makedirs(plots_directory)
    if not os.path.exists(plots_directory + 'eps/'):
        os.makedirs(plots_directory + 'eps/')
    #if not creating heatmap for all methods, filter the matrix and save into separate file

    methodList = methods
    gnuplotHeatMapFile = matrix_file.replace('txt','gnuplot')
    template_values = {
        'title': '',
        'input_file': '../../%s' % matrix_file,
        'output_file': './eps/'  + matrix_file.replace('.txt',''),
        'row_count': len(methodList),
        'row_descriptions': ','.join(['"%s" %s' % (methods[i].replace('_','-').replace('snapshots','sn').replace('aggregated','ag'),i)  for i in range(0,len(methods)) if methods[i] in methodList])
    }
    open(plots_directory + gnuplotHeatMapFile, mode='wb').write(gnuplotHeatmapTemplate % template_values)

    p = subprocess.Popen(['gnuplot', gnuplotHeatMapFile], stdout=subprocess.PIPE, cwd = plots_directory)
    print p.stdout.read()


    return 'intersections/eps/' + matrix_file.replace('txt','eps')

generateReport = True

#for the next report parts, in case there are multiple construction params, choose only one
default_aggregated_params = 'aggregated_1'
#default_snapshot_params = 'snapshots_1month_1'
default_snapshot_params = 'snapshots_5_1'

include_random_results = True
distance_measure_list = ['pca_2']
ignore_methods = []
ignore_methods_for_intersection_matrix = []
include_intersection_matrix = True
use_ignored_pairs_ground_truth_results = False
save_summary_values = False


opts, args = getopt.getopt(sys.argv[1:], "s:d:p:rh", ["subdir=","help","default-snapshot-params=","default-aggregated-params=","ignore-methods=","save-summary-values","ignore-methods-for-intersection-matrix=","do-not-include-intersection-matrix","use-ignored-pairs-ground-truth-results"])
for o, a in opts:
    if o == '-d':
        distance_measure_list = a.split(',')
    elif o ==  '--default-snapshot-params':
        default_snapshot_params = a
    elif o == '--default-aggregated-params':
        default_aggregated_params = a
    elif o in ('--save-summary-values'):
        save_summary_values = True
    elif o in ('--ignore-methods'):
        ignore_methods = a.split(',')
    elif o in ('--ignore-methods-for-intersection-matrix'):
        ignore_methods_for_intersection_matrix = a.split(',')
    elif o in ('--do-not-include-intersection-matrix '):
        include_intersection_matrix = False
    elif o in ('--use-ignored-pairs-ground-truth-results'):
        use_ignored_pairs_ground_truth_results = True
    elif o in ('-h','--help'):
        print """Usage:
%s analysis_type  ground_truth_type
analysis_type can be either 'model' for model analysis or network name for node ground truth analysis
-d                                          Distance measure list (by default: pca_2)
--default-snapshot-params                   Default snapshot network construction params for report (in case there are several of them)
--default-aggregated-params                 Default aggregated network construction params for report (in case there are several of them)
--save-summary-values                       Save summary values into a spreadsheet
--ignore-methods                            Methods to ignore (if method name contains any of the input substrings, ignore it)
--ignore-methods-for-intersection-matrix    Methods to ignore for intersection matrix (if method name contains any of the input substrings, ignore it)
--do-not-include-intersection-matrix        Do not include intersection matrix into the report
--use-ignored-pairs-ground-truth-results    Use results where object pairs with certain ground truth values were ignored (i.e., '_ignoring_pr_roc')
-h,--help                                   Print this""" % sys.argv[0]
        sys.exit(0)

analysis_type = args[0]
analysis_type_sub_dir_list = args[1].split(',')

analysis_dir = settings.BASE_OUTPUT_DIR
if analysis_type == 'model':
    analysis_dir += 'MODEL_ANALYSIS/'
else:
    analysis_dir += '%s/pca/' % (analysis_type)

if use_ignored_pairs_ground_truth_results:
    print 'Use results where object pairs with certain ground truth values were ignored'



a_counter = 1
for analysis_type_sub_dir in analysis_type_sub_dir_list:
    print '%s/%s. %s' % (a_counter,len(analysis_type_sub_dir_list), analysis_type_sub_dir)
    a_counter += 1
    d_counter = 1

    for distance_measure in distance_measure_list:
        print '%s/%s. %s' % (d_counter,len(distance_measure_list), distance_measure)
        d_counter += 1

        if not use_ignored_pairs_ground_truth_results:
           distances_dir = analysis_dir + '%s/%s_distances/' % (analysis_type_sub_dir,distance_measure)
        else:
           distances_dir = analysis_dir + '%s/%s_distances_ignoring/' % (analysis_type_sub_dir,distance_measure)


        methods = gdv_utils.getPcaAnalysisPrRocMethods(analysis_type, analysis_type_sub_dir,distance_measure,use_ignored_pairs_ground_truth_results)

        if len(ignore_methods)>0:
            print 'Ignoring methods: %s' % ignore_methods
            for m in ignore_methods:
                  for method in methods[:]:
                      if (m in method) and (method in methods):
                          methods.remove(method)

        print 'Including the following %s methods: %s' % (len(methods), ', '.join(methods))
        methodsDetails = [gdv_utils.getMethodDetailsDict(m) for m in methods]

        auprs = [pr_roc_utils.get_AUPR(distances_dir + '%s_pr_roc.txt' % m) for m in methods]
        aurocs = [pr_roc_utils.get_AUROC(distances_dir + '%s_pr_roc.txt' % m) for m in methods]
        break_evens = [pr_roc_utils.get_break_even_point(distances_dir + '%s_pr_roc.txt' % m) for m in methods]
        max_fscores = [pr_roc_utils.get_max_fscore(distances_dir + '%s_pr_roc.txt' % m) for m in methods]
        if include_random_results:
            file_list = []
            files_dir = distances_dir + 'random/'
            for f in os.listdir(files_dir):
                if 'random' in f and 'pr_roc' in f:
                    file_list.append(files_dir + f)

            random_auprs =  [pr_roc_utils.get_AUPR(f) for f in file_list]
            random_aurocs = [pr_roc_utils.get_AUROC(f) for f in file_list]

            random_breakevens = [pr_roc_utils.get_break_even_point(f) for f in file_list]
            random_breakevens_b = [b[0] for b in random_breakevens]
            random_breakevens_k = [b[1] for b in random_breakevens]

            random_maxfscores = [pr_roc_utils.get_max_fscore(f) for f in file_list]
            random_maxfscores_f = [f[0] for f in random_maxfscores]
            random_maxfscores_k = [f[1] for f in random_maxfscores]

            auprs += [np.average(random_auprs),np.std(random_auprs)]
            aurocs += [np.average(random_aurocs),np.std(random_aurocs)]
            break_evens.append(pr_roc_utils.get_break_even_point(distances_dir + 'random_pr_roc.txt',contains_std_dev=True))
            max_fscores.append(pr_roc_utils.get_max_fscore(distances_dir + 'random_pr_roc.txt',contains_std_dev=True))
            #break_evens.append((np.average(random_breakevens_b),np.average(random_breakevens_k)))
            #max_fscores.append((np.average(random_maxfscores_f),np.average(random_maxfscores_k)))
            #random_breakevens_b = [b for b in random_breakevens_b if b>0]
            #random_maxfscores_f = [b for b in random_maxfscores_f if b>0]
            #print 'Random break-even: %s (%s)' % (np.average(random_breakevens_b),np.std(random_breakevens_b))
            #print 'Random max f-score: %s (%s)' % (np.average(random_maxfscores_f),np.std(random_maxfscores_f))

        #save AUPRs
        aupr_filename = distances_dir + 'aupr_values.txt'
        aupr_filename_relative = '../../aupr_values.txt' #for gnuplot
        output_file = open(aupr_filename,'wb')
        output_file.write('\t'.join([str(a) + '\t0' for a in auprs[:-2]])) # added zeros as std (so that to show later std for random method)
        output_file.write('\t' + '\t'.join([str(a) for a in auprs[-2:]]))
        output_file.close()

        #save AUROCs
        auroc_filename = distances_dir + 'auroc_values.txt'
        auroc_filename_relative = '../../auroc_values.txt' #for gnuplot
        output_file = open(auroc_filename,'wb')
        output_file.write('\t'.join([str(a) + '\t0' for a in aurocs[:-2]]))
        output_file.write('\t' + '\t'.join([str(a) for a in aurocs[-2:]]))
        output_file.close()

        if save_summary_values:    #save summary values
            print 'Saving summary values...'
            summary_filename = distances_dir + '%s_%s%s_summary_values.csv' % (analysis_type, analysis_type_sub_dir,'' if not use_ignored_pairs_ground_truth_results else '_ignoring')
            output_file = open(summary_filename,'wb')
            output_file.write('Method,AUPR,Std. AUPR,AUROC,Std. AUROC,Break-even point,Break-even k,Max F-score,Max F-score k,AUPR,Std. AUPR,AUROC,Std. AUROC\n')
            for i in range(0,len(methods)):
                sys.stdout.write('\ri=%s/%s...' % (i+1,len(methods)))
                sys.stdout.flush()
                output_file.write('%s,%s,-,%s,-,%s,%.5f,%s,%.5f' % (methods[i], auprs[i], aurocs[i],break_evens[i][0],break_evens[i][1],max_fscores[i][0],max_fscores[i][1]))
                #add random shuffling results
                if os.path.exists(distances_dir + 'random_shuffle/'):
                    for f in os.listdir(distances_dir + 'random_shuffle/'):
                        if methods[i] in f and 'pr_roc' in f:
                            file_list.append(distances_dir + 'random_shuffle/' + f)
                    if len(file_list)>0:
                        random_auprs =  [pr_roc_utils.get_AUPR(f) for f in file_list]
                        random_aurocs = [pr_roc_utils.get_AUROC(f) for f in file_list]
                        output_file.write(',%s,%s,%s,%s' % (np.average(random_auprs),np.std(random_auprs),np.average(random_aurocs),np.std(random_aurocs)))

                output_file.write('\n')

            if include_random_results:
                output_file.write('random,%s,%s,%s,%s,%s,%.5f,%s,%.5f,%s,%s,%s,%s\n' % (auprs[-2], auprs[-1], aurocs[-2], aurocs[-1],
                                                                                        break_evens[-1][0],break_evens[-1][1],max_fscores[-1][0],max_fscores[-1][1],
                                                                                        auprs[-2], auprs[-1], aurocs[-2], aurocs[-1]))
            output_file.close()

            if analysis_type == 'aging':
                if not os.path.exists(distances_dir + 'candidate_proteins/'):
                    os.makedirs(distances_dir + 'candidate_proteins/')
                aging_labels = ground_truth_utils.get_node_labels(analysis_type,ground_truth_type=analysis_type_sub_dir)

                max_k = 10
                print 'Saving first %s TP/FP/Precision/Recall values for methods...' % max_k
                k_values_filename = distances_dir + '%s_%s%s_methods_k_values_%s.csv' % (analysis_type, analysis_type_sub_dir,'' if not use_ignored_pairs_ground_truth_results else '_ignoring',distance_measure)
                output_file = open(k_values_filename,'wb')

                output_file.write('Method')
                for k in range(0,max_k):
                    output_file.write(',k_%s,TP,FP,P,Precision,Recall,F-score' % (k+1))
                output_file.write('\n')

                k_col = pr_roc_utils.get_pr_roc_file_column_number('k',starting_from_one=False)
                tp_col = pr_roc_utils.get_pr_roc_file_column_number('tp',starting_from_one=False)
                fp_col =  pr_roc_utils.get_pr_roc_file_column_number('fp',starting_from_one=False)
                p_col = pr_roc_utils.get_pr_roc_file_column_number('p',starting_from_one=False)
                r_col = pr_roc_utils.get_pr_roc_file_column_number('r',starting_from_one=False)
                f_col = pr_roc_utils.get_pr_roc_file_column_number('f',starting_from_one=False)

                for m in methods:
                    #if m not in ['aggregated_3_sgdvs_5','stgdvs_3','dgdvs_5_4_1','dgdvs_5_3_1','dgdvs_7_3_1','dcgdvs_8_3_1']:
                    #    continue
                    with open(distances_dir + '%s_pr_roc.txt' % m,'r') as pr_roc_file:
                        output_file.write('%s' % m)
                        i=1
                        for line in pr_roc_file.readlines()[2:max_k+2]:
                            values = line.split('\t')
                            tp = int(values[tp_col])
                            fp = int(values[fp_col])
                            total = tp + fp
                            output_file.write(',%s,%s,%s,%s,%s,%s,%s' % (values[k_col],tp,fp,total,values[p_col],values[r_col],values[f_col]))

                            #Saving protein candidates lists
                            # pairs = pca_utils.get_top_k_pairs_from_distance_file(analysis_dir + '%s_distances%s_%s.txt' % (m,'' if not use_ignored_pairs_ground_truth_results else '_ignoring', distance_measure),total)
                            # proteins = []
                            # for p in pairs:
                            #     if aging_labels[p[0]] == aging_labels[p[1]]:#need one aging and one non-aging
                            #         continue
                            #     proteins.append(p[0] if aging_labels[p[0]]==0 else p[1]) # need non-aging as a candidate
                            # proteins = utils.remove_duplicates_without_changing_order(proteins)
                            # f = open(distances_dir + 'candidate_proteins/%s_list_%s.txt' % (m,i),'wb')
                            # f.write('\n'.join(proteins))
                            # f.close()
                            i+=1

                        output_file.write('\n')

                #calculate protein candidate lists pairwise intersections
                # intersection_protein_lists_methods = ['aggregated_3_sgdvs_5','stgdvs_3','dgdvs_5_4_1','dgdvs_5_3_1','dgdvs_7_3_1','dcgdvs_8_3_1']
                # for k in range(1,max_k+1):
                #     intersection_matrix = np.eye(len(intersection_protein_lists_methods))
                #     i_pair = 1
                #
                #     for i in range(0,len(intersection_protein_lists_methods)-1):
                #         method1 = intersection_protein_lists_methods[i]
                #         list1 = open(distances_dir + 'candidate_proteins/%s_list_%s.txt' % (method1,k),'r').readlines()
                #
                #         for j in range(i+1,len(intersection_protein_lists_methods)):
                #             method2 = intersection_protein_lists_methods[j]
                #             list2 = open(distances_dir + 'candidate_proteins/%s_list_%s.txt' % (method2,k),'r').readlines()
                #
                #             #compute Jaccard similarity
                #             similarity = 1.0 * len(utils.get_intersection(list1,list2))
                #             similarity /= (len(list1) + len(list2) - similarity)
                #
                #             intersection_matrix[i,j] = intersection_matrix[j,i] = similarity
                #
                #     df = pd.DataFrame(intersection_matrix, index=intersection_protein_lists_methods, columns=intersection_protein_lists_methods)
                #     df.to_csv(distances_dir + 'candidate_proteins/intersections_%s.csv' % (k), index=True, header=True, sep=',')

                if include_random_results:
                    k_col = pr_roc_utils.get_pr_roc_file_column_number('k',starting_from_one=False,contains_std_dev=True)
                    tp_col = pr_roc_utils.get_pr_roc_file_column_number('tp',starting_from_one=False,contains_std_dev=True)
                    fp_col =  pr_roc_utils.get_pr_roc_file_column_number('fp',starting_from_one=False,contains_std_dev=True)
                    p_col = pr_roc_utils.get_pr_roc_file_column_number('p',starting_from_one=False,contains_std_dev=True)
                    r_col = pr_roc_utils.get_pr_roc_file_column_number('r',starting_from_one=False,contains_std_dev=True)
                    f_col = pr_roc_utils.get_pr_roc_file_column_number('f',starting_from_one=False,contains_std_dev=True)
                    with open(distances_dir + 'random_pr_roc.txt') as pr_roc_file:
                        output_file.write('random')
                        for line in pr_roc_file.readlines()[2:max_k+2]:
                            values = line.split('\t')
                            tp = float(values[tp_col])
                            fp = float(values[fp_col])
                            total = tp + fp
                            output_file.write(',%s,%s,%s,%s,%s,%s,%s' % (values[k_col],tp,fp,total,values[p_col],values[r_col],values[f_col]))
                        output_file.write('\n')
                output_file.close()


        #TODO: extract method groupping function into separate functions and reuse them
        if generateReport:
            #group plots so that to answer different questions

            #generate latex summary
            print 'Generating summary report...'

            analysis_type_header = ''
            if analysis_type == 'model':
                analysis_type_header = 'Model network analysis'
            else:
                analysis_type_header = '%s node ground truth analysis' % analysis_type

            report = LatexBuilder(
                author = 'Yuriy Hulovatyy',
                title = '%s report for \'%s\' with %s distance  ' %  (analysis_type_header, analysis_type_sub_dir.replace('_','-'),distance_measure.replace('_','-')),
                graphicspath= ['../../plots/','../../../pca/plots/eps/' if analysis_type == 'model' else '../../../plots/eps/']
                )

            report_name = 'report_%s_%s_model_analysis' % (analysis_type_sub_dir,distance_measure)

            #------------------------------------------------------------------------------------------------------

            #correspondence in sizes between static and dynamic graphlets
            grahplet_size_correspondence_dict = {
                '3_3': 3,
                '6_4': 4,
                '10_5': 5,
            }

            methods_temp_dict = {
                3: {},
                4: {},
                5: {}
            }


            any_relevant_methods = False

            for m in methodsDetails:

                mKey = '%sgdvs' % m['type']
                size_key = None
                if m['type'] in ['d','dc','ds','dcs']:
                    mKey += '_%s_%s_%s' % (m['k'], m['n'], m['delta_t'])
                    if grahplet_size_correspondence_dict.has_key('%s_%s' % (m['k'], m['n'])):
                        size_key =  grahplet_size_correspondence_dict['%s_%s' % (m['k'], m['n'])]
                else:
                    mKey += '_%s' % m['n']
                    if methods_temp_dict.has_key(m['n']):
                        size_key = m['n']
                #print 'mkey-- %s' %mKey
                if size_key is None:
                    continue
                elif not methods_temp_dict.has_key(size_key):
                    methods_temp_dict[size_key] = {}

                if methods_temp_dict[size_key].has_key(mKey):
                    methods_temp_dict[size_key][mKey].append(m['network_construction'])
                else:
                    methods_temp_dict[size_key][mKey] = [m['network_construction']]

                if len(methods_temp_dict[size_key].keys())>1:
                    any_relevant_methods = True

            #------------------------------------------------------------------------------------------------------
            print 'Part 1: Static vs dynamic...'


            if not any_relevant_methods:
                print 'No methods to compare for static vs dynamic'
            else:
                methodsForHeatmap = []
                report.addSection('Static vs dynamic',0, clearPage=True)
                #remove redundant network constructions
                for size_key in methods_temp_dict.keys():

                    for mKey in methods_temp_dict[size_key]:
                    #print 'method tmp dict: %s. | size key: %s | mkey: %s' %(methods_temp_dict[size_key][mKey],size_key,mKey)
                        if len(methods_temp_dict[size_key][mKey])>1:
                            if 'st' in mKey or 'dgdvs' in mKey or 'dcgdvs' in mKey or 'dsgdvs' in mKey or 'dcsgdvs' in mKey:
                                methods_temp_dict[size_key][mKey] = [default_snapshot_params]
                            else:
                                methods_temp_dict[size_key][mKey] = [default_aggregated_params]

                    if len(methods_temp_dict[size_key])<2:
                        continue

                    report.addSection('%s-graphlets' % size_key,1)

                    method_names_sorted = sorted(methods_temp_dict[size_key], cmp = gdv_utils.compareMethodNames)
                    method_names_sorted = [(methods_temp_dict[size_key][m1][0] + '_' +  m1) if methods_temp_dict[size_key][m1][0] != '' else m1 for m1 in method_names_sorted]


                    plot_list = [
                    getLinePlot(distances_dir, method_names_sorted, 'r', 'p',  output_suffix='s-vs-d-%s' % size_key),
                    getLinePlot(distances_dir, method_names_sorted, 'fp_rate', 'tp_rate',  output_suffix='s-vs-d-%s' % size_key),
                    getLinePlot(distances_dir, method_names_sorted, 'k', 'f', output_suffix='s-vs-d-%s' % size_key),
                    getBarPlot(distances_dir, method_names_sorted,'Methods','AUPR', aupr_filename_relative, methods, output_suffix='s-vs-d-%s' % size_key),
                    getBarPlot(distances_dir, method_names_sorted,'Methods','AUROC', auroc_filename_relative, methods, output_suffix='s-vs-d-%s' % size_key),
                    ]

                    image_caption_list = [
                        'Precision-Recall',
                        'ROC curve',
                        'F-score',
                        'AUPR',
                        'AUROC'
                    ]


                    report.addGroupFigure('Static vs dynamics for graphlets of size %s' % size_key,
                          plot_list,
                          image_caption_list = image_caption_list,
                          image_angle=270,
                          column_count=3
                          )

            #------------------------------------------------------------------------------------------------------
            print 'Part 2: Graphlet size...'
            any_relevant_methods = False
            methods_temp_dict = dict()
            for m in methodsDetails:
                m_type = m['type']
                if 's' in m_type:
                    mKey = '' #for s and st methods, we can vary only n
                else:
                    mKey = '%s_%s' % (m['k'], m['delta_t'])

                if not methods_temp_dict.has_key(m_type):
                    methods_temp_dict[m_type] = {}

                if methods_temp_dict[m_type].has_key(mKey):
                    methods_temp_dict[m_type][mKey].append(m['full_name'])
                    any_relevant_methods = True
                else:
                    methods_temp_dict[m_type][mKey] = [m['full_name']]

            #choose one network construction (if there are more than one)
            for m_type in methods_temp_dict:
                for mKey in methods_temp_dict[m_type]:
                    constr_parms = set()

                    for m in methods_temp_dict[m_type][mKey]:
                        m_temp  = gdv_utils.getMethodDetailsDict(m)
                        constr_parms.add(m_temp['network_construction'])
                    if len(constr_parms)>1:
                        m_list_temp  = list(methods_temp_dict[m_type][mKey])
                        for m in m_list_temp:
                            m_temp = gdv_utils.getMethodDetailsDict(m)
                            if m_type != 's' and default_snapshot_params != m_temp['network_construction']:
                                methods_temp_dict[m_type][mKey].remove(m)
                            elif m_type == 's' and default_aggregated_params != m_temp['network_construction']:
                                methods_temp_dict[m_type][mKey].remove(m)

            if not any_relevant_methods:
                print 'No methods to compare for the effect of graphlet size'
            else:
                report.addSection('Graphlet number of nodes',0, clearPage=True)
                for m_type in sorted(methods_temp_dict.keys(), cmp = gdv_utils.compareMethodTypes):

                    image_caption_list = []
                    plotList = []

                    if len(methods_temp_dict[m_type])>0:
                        report.addSection(gdv_utils.getMethodTypeDescr(m_type),1)

                    for mKey in methods_temp_dict[m_type]:
                            #compare_key - for s and st only one; for dynamic methods - number of events + delta t
                            methods_to_compare = methods_temp_dict[m_type][mKey]

                            if len(methods_to_compare)<2:
                                continue

                            compare_key_str = '_%s' % mKey if mKey != '' else ''

                            plot_list = [
                                getLinePlot(distances_dir, methods_to_compare, 'r', 'p', output_suffix='n-size-%s%s' % (m_type,compare_key_str)),
                                getLinePlot(distances_dir, methods_to_compare, 'fp_rate', 'tp_rate', output_suffix='n-size-%s%s' % (m_type,compare_key_str)),
                                getLinePlot(distances_dir, methods_to_compare, 'k', 'f', output_suffix='n-size-%s%s' % (m_type,compare_key_str)),
                                getBarPlot(distances_dir, methods_to_compare,'Methods','AUPR', aupr_filename_relative, methods, output_suffix='n-size-%s%s' % (m_type,compare_key_str)),
                                getBarPlot(distances_dir, methods_to_compare,'Methods','AUROC', auroc_filename_relative, methods, output_suffix='n-size-%s%s' % (m_type,compare_key_str)),
                            ]

                            image_caption_list = [
                                'Precision-Recall',
                                'ROC curve',
                                'F-score',
                                'AUPR',
                                'AUROC'
                            ]

                            if mKey != '':
                                compare_key_str = ' (k=%s, t=%s)' % (mKey.split('_')[0],mKey.split('_')[1])

                            report.addGroupFigure('Graphlet size for %s%s' % (gdv_utils.getMethodTypeDescr(m_type),compare_key_str),
                                  plot_list,
                                  image_caption_list = image_caption_list,
                                  image_angle=270,
                                  column_count=3
                                  )

            #------------------------------------------------------------------------------------------------------
            print 'Part 3: Number of events...'
            any_relevant_methods = False


            methods_temp_dict = {}
            for m in methodsDetails:
                if 's' in  m['type']:
                    continue
                mKey = '%s_%s' % (m['n'], m['delta_t'])

                if not methods_temp_dict.has_key(m['type']):
                    methods_temp_dict[m['type']] = {}

                if methods_temp_dict[m['type']].has_key(mKey):
                    methods_temp_dict[m['type']][mKey].append(m['full_name'])
                    any_relevant_methods = True
                else:
                    methods_temp_dict[m['type']][mKey] = [m['full_name']]

            #choose one network construction (if there are more than one)
            for m_type in methods_temp_dict:
                for mKey in methods_temp_dict[m_type]:
                    constr_parms = set()

                    for m in methods_temp_dict[m_type][mKey]:
                        m_temp  = gdv_utils.getMethodDetailsDict(m)
                        constr_parms.add(m_temp['network_construction'])
                    if len(constr_parms)>1:
                        m_list_temp  = list(methods_temp_dict[m_type][mKey])
                        for m in m_list_temp:
                            m_temp = gdv_utils.getMethodDetailsDict(m)
                            if default_snapshot_params != m_temp['network_construction']:
                                methods_temp_dict[m_type][mKey].remove(m)

            if not any_relevant_methods:
                print 'No methods to compare for the effect of graphlet size'
            else:
                report.addSection('Graphlet number of events',0, clearPage=True)
                for m_type in sorted(methods_temp_dict.keys(), cmp = gdv_utils.compareMethodTypes):

                    if len(methods_temp_dict[m_type]) == 0:
                        continue

                    report.addSection(gdv_utils.getMethodTypeDescr(m_type),1)



                    for mKey in sorted(methods_temp_dict[m_type]):
                        if len(methods_temp_dict[m_type][mKey])<2:
                            continue

                        image_caption_list = []
                        plotList = []
                        temp_methods_list = sorted(methods_temp_dict[m_type][mKey], gdv_utils.compareMethodNames)
                        #print ', '.join(temp_methods_list)

                        plot_list = [
                            getLinePlot(distances_dir, temp_methods_list, 'r', 'p', output_suffix='k-size-%s-%s' % (m_type,mKey.split('_')[0])),
                            getLinePlot(distances_dir, temp_methods_list, 'fp_rate', 'tp_rate', output_suffix='k-size-%s-%s' % (m_type,mKey.split('_')[0])),
                            getLinePlot(distances_dir, temp_methods_list, 'k', 'f', output_suffix='k-size-%s-%s' % (m_type,mKey.split('_')[0])),
                            getBarPlot(distances_dir, temp_methods_list,'Methods','AUPR', aupr_filename_relative, methods, output_suffix='k-size-%s-%s' % (m_type,mKey.split('_')[0])),
                            getBarPlot(distances_dir, temp_methods_list,'Methods','AUROC', auroc_filename_relative, methods, output_suffix='k-size-%s-%s' % (m_type,mKey.split('_')[0])),
                        ]

                        image_caption_list = [
                            'Precision-Recall',
                            'ROC curve',
                            'F-score',
                            'AUPR',
                            'AUROC'
                        ]


                        report.addGroupFigure('Comparison for %s (n=%s)' % (gdv_utils.getMethodTypeDescr(m_type),mKey.split('_')[0]),
                              plot_list,
                              image_caption_list = image_caption_list,
                              image_angle=270,
                              column_count=3
                              )



            print 'Part 4: Network construction type'

            methods_temp_dict = {}
            for m in methodsDetails:
                mKey = '%sgdvs' % m['type']
                if 'd' in m['type']:
                    mKey += '_%s_%s_%s' % (m['k'], m['n'], m['delta_t'])
                else:
                    mKey += '_%s' % m['n']

                if methods_temp_dict.has_key(mKey):
                    methods_temp_dict[mKey].append(m['network_construction'])
                    any_relevant_methods = True
                else:
                    methods_temp_dict[mKey] = [m['network_construction']]

            if  not any_relevant_methods:
                print 'No methods for analysis'
            else:
                mKeyList = sorted(methods_temp_dict.keys(), cmp = gdv_utils.compareMethodNames)
                prev_method_type = ''
                added_section_title = False

                for mKey in mKeyList:
                    if len(methods_temp_dict[mKey])<2:
                        continue

                    if not added_section_title:#in case there are no methods with different network construction types, do not add section title
                        report.addSection('Network construction',0,clearPage=True)
                        added_section_title = True

                    current_method_type = gdv_utils.getMethodName(mKey, onlyType=True)
                    if current_method_type != prev_method_type:
                        report.addSection(gdv_utils.getMethodName(mKey, onlyType=True),1)
                    prev_method_type = current_method_type

                    image_caption_list = []
                    plotList = []

                    temp_methods_list = sorted([c + '_' + mKey  for c in sorted(methods_temp_dict[mKey])], gdv_utils.compareMethodNames)

                    plot_list = [
                            getLinePlot(distances_dir, temp_methods_list, 'r', 'p', output_suffix='net-constr-%s' % mKey),
                            getLinePlot(distances_dir, temp_methods_list, 'fp_rate', 'tp_rate', output_suffix='net-constr-%s' % mKey),
                            getLinePlot(distances_dir, temp_methods_list, 'k', 'f', output_suffix='net-constr-%s' % mKey),
                            getBarPlot(distances_dir, temp_methods_list,'Methods','AUPR', aupr_filename_relative, methods, output_suffix='net-constr-%s' % mKey),
                            getBarPlot(distances_dir, temp_methods_list,'Methods','AUROC', auroc_filename_relative, methods, output_suffix='net-constr-%s' % mKey),
                        ]

                    image_caption_list = [
                            'Precision-Recall',
                            'ROC curve',
                            'F-score',
                            'AUPR',
                            'AUROC'
                        ]

                    report.addGroupFigure('Comparison for %s' % (gdv_utils.getMethodName(mKey)),
                              plot_list,
                              image_caption_list = image_caption_list,
                              image_angle=270,
                              column_count=3
                              )

            print 'Part 5: Individual'

            if len(methods) == 0:
                print 'No methods for analysis'
            else:
                report.addSection('Individual',0,clearPage=True)

                methods_temp_dict = {}
                for m in methodsDetails:
                    if not methods_temp_dict.has_key(m['type']):
                        methods_temp_dict[m['type']] = []
                    methods_temp_dict[m['type']].append(m['full_name'])

                #Precision/Recall vs k
                report.addSection('Precision/Recall vs k',1)
                for m_type in sorted(methods_temp_dict.keys(), cmp = gdv_utils.compareMethodTypes):
                    if len(methods_temp_dict[m_type])==0:
                        continue

                    report.addSection(gdv_utils.getMethodTypeDescr(m_type),2)
                    plot_list = []
                    image_caption_list = []
                    for m in sorted(list(set(methods_temp_dict[m_type]))):
                        plot_list.append(getLinePlot(distances_dir, m, 'k', ['p','r'], m))
                        image_caption_list.append(gdv_utils.getMethodName(m))

                    report.addGroupFigure('Results for %s' % (gdv_utils.getMethodTypeDescr(m_type)),
                                          plot_list,
                                          image_caption_list = image_caption_list,
                                          image_angle=270,
                                          column_count=3
                                          )

                #PCA figures
                # report.addSection('PCA Figures',1)
                # for m_type in sorted(methods_temp_dict.keys(), cmp = gdv_utils.compareMethodTypes):
                #     if len(methods_temp_dict[m_type])==0:
                #         continue
                #
                #     report.addSection(gdv_utils.getMethodTypeDescr(m_type),2)
                #     plot_list = []
                #     image_caption_list = []
                #     for m in sorted(list(set(methods_temp_dict[m_type])), cmp = gdv_utils.compareMethodNames):
                #         plot_list.append('%s.eps' % m)
                #         image_caption_list.append(gdv_utils.getMethodName(m))
                #
                #
                #     report.addGroupFigure('Results for %s' % (gdv_utils.getMethodTypeDescr(m_type)),
                #                           plot_list,
                #                           image_caption_list = image_caption_list,
                #                           image_angle=0,
                #                           column_count=4,
                #                           max_images_per_figure = 12
                #                           )



            print 'Part 6: Best Methods'

            if len(methods) == 0:
                print 'No methods for comparison'
            else:
                report.addSection('Best methods',0,clearPage=True)

                methods_temp_dict = {}
                for m in methodsDetails:
                    if not methods_temp_dict.has_key(m['type']):
                        methods_temp_dict[m['type']] = []
                    methods_temp_dict[m['type']].append(m['full_name'])

                #for static-temporal and dynamic compare across consistent network constructions
                methods_temp_network_construction_dict = {}
                for m in methodsDetails:
                    if m['type'] == 's':
                        continue

                    if not methods_temp_network_construction_dict.has_key(m['network_construction']):
                        methods_temp_network_construction_dict[m['network_construction']] = {}

                    if not methods_temp_network_construction_dict[m['network_construction']].has_key(m['type']):
                        methods_temp_network_construction_dict[m['network_construction']][m['type']] = []
                    methods_temp_network_construction_dict[m['network_construction']][m['type']].append(m['full_name'])


                if len(methods_temp_network_construction_dict)<2:#if less than two different dynamic network construction, no need for separate evaluation
                    methods_temp_network_construction_dict = {}

                for network_construction_type in ['all'] + methods_temp_network_construction_dict.keys():
                    report.addSection(network_construction_type.capitalize(),1)
                    for criterion_for_best_method in ['aupr','auroc']:
                        report.addSection('According to %s' % criterion_for_best_method.upper(),2)

                        best_methods = [] #one method for each type
                        for m_type in sorted(methods_temp_dict.keys(), cmp = gdv_utils.compareMethodTypes):
                            if len(methods_temp_dict[m_type])==0:
                                continue

                            best_value = -1
                            best_method = []

                            available_methods = []

                            if m_type == 's' or network_construction_type == 'all':#consider best of all combinations
                                available_methods = methods_temp_dict[m_type]
                            elif methods_temp_network_construction_dict[network_construction_type].has_key(m_type):#consider best of the same network construction
                                available_methods = methods_temp_network_construction_dict[network_construction_type][m_type]

                            for m in available_methods:
                                value = auprs[methods.index(m)] if criterion_for_best_method == 'aupr' else aurocs[methods.index(m)] if criterion_for_best_method == 'auroc' else None

                                if value >= best_value:
                                    best_value = value
                                    #if value == best_value:
                                    #    best_method.append(m) #there can be several best methods for a category
                                    #else:
                                    best_method = [m]

                            best_methods += best_method

                        if include_random_results:
                            best_methods.insert(0,'random')

                        plot_list = [
                            getLinePlot(distances_dir, best_methods, 'r', 'p', output_suffix='best_%s_by_%s' % (network_construction_type,criterion_for_best_method)),
                            getLinePlot(distances_dir, best_methods, 'fp_rate', 'tp_rate', output_suffix='best_%s_by_%s' % (network_construction_type,criterion_for_best_method)),
                            getLinePlot(distances_dir, best_methods, 'k', 'f', output_suffix='best_%s_by_%s' % (network_construction_type,criterion_for_best_method)),
                            getBarPlot(distances_dir, best_methods, 'Methods','AUPR', aupr_filename_relative, methods, output_suffix='best_%s_by_%s' % (network_construction_type,criterion_for_best_method)),
                            getBarPlot(distances_dir, best_methods, 'Methods','AUROC', auroc_filename_relative, methods, output_suffix='best_%s_by_%s' % (network_construction_type,criterion_for_best_method)),
                        ]

                        image_caption_list = [
                            'Precision-Recall',
                            'ROC curve',
                            'F-score',
                            'AUPR',
                            'AUROC'
                        ]

                        report.addGroupFigure('Comparison of the best methods',
                              plot_list,
                              image_caption_list = image_caption_list,
                              image_angle=270,
                              column_count=3
                              )



                        #e.g.,if the best static is sgdvs_3 -> compare with dgdvs_3_3_1 and so on.
                        best_static_methods = [m for m in best_methods if 'sgdvs' in m]
                        if len(best_static_methods)>0:
                            best_static_method = best_static_methods[0]
                            best_static_method_n = gdv_utils.getMethodDetailsDict(best_static_method)['n']
                            best_methods = [best_static_method]
                            for m_type in sorted(methods_temp_dict.keys(), cmp = gdv_utils.compareMethodTypes):
                                if len(methods_temp_dict[m_type])==0:
                                    continue
                                if m_type == 's':
                                    continue
                                best_value = -1
                                best_method = []

                                available_methods = []

                                if m_type == 's' or network_construction_type == 'all':#consider best of all combinations
                                    available_methods = methods_temp_dict[m_type]
                                elif methods_temp_network_construction_dict[network_construction_type].has_key(m_type):#consider best of the same network construction
                                    available_methods = methods_temp_network_construction_dict[network_construction_type][m_type]

                                for m in available_methods:
                                    m_details_dict = gdv_utils.getMethodDetailsDict(m)

                                    if m_type == 'st' and m_details_dict['n']!=best_static_method_n:
                                        continue
                                    elif m_type in ['d','dc'] and grahplet_size_correspondence_dict.get('%s_%s' % (m_details_dict['k'],m_details_dict['n']),-1)!=best_static_method_n:
                                        continue

                                    value = auprs[methods.index(m)] if criterion_for_best_method == 'aupr' else aurocs[methods.index(m)] if criterion_for_best_method == 'auroc' else None
                                    if value >= best_value:
                                        best_value = value
                                        #if value == best_value:
                                        #    best_method.append(m) #there can be several best methods for a category
                                        #else:
                                        best_method = [m]

                                best_methods += best_method

                            best_methods = sorted(list(set(best_methods)), cmp = gdv_utils.compareMethodNames)

                            if include_random_results:
                                best_methods.insert(0,'random')

                            plot_list = [
                                getLinePlot(distances_dir, best_methods, 'r', 'p', output_suffix='best_direct_counterparts_%s_by_%s' % (network_construction_type,criterion_for_best_method)),
                                getLinePlot(distances_dir, best_methods, 'fp_rate', 'tp_rate', output_suffix='best_direct_counterparts_%s_by_%s' % (network_construction_type,criterion_for_best_method)),
                                getLinePlot(distances_dir, best_methods, 'k', 'f', output_suffix='best_direct_counterparts_%s_by_%s' % (network_construction_type,criterion_for_best_method)),
                                getBarPlot(distances_dir, best_methods, 'Methods','AUPR', aupr_filename_relative, methods, output_suffix='best_direct_counterparts_%s_by_%s' % (network_construction_type,criterion_for_best_method)),
                                getBarPlot(distances_dir, best_methods, 'Methods','AUROC', auroc_filename_relative, methods, output_suffix='best_direct_counterparts_%s_by_%s' % (network_construction_type,criterion_for_best_method)),
                            ]

                            image_caption_list = [
                                'Precision-Recall',
                                'ROC curve',
                                'F-score',
                                'AUPR',
                                'AUROC'
                            ]

                            report.addGroupFigure('Comparison of the best direct static counterparts',
                                  plot_list,
                                  image_caption_list = image_caption_list,
                                  image_angle=270,
                                  column_count=3
                                  )

            print 'Part 7: Pairwise Intersections'
            intersection_sizes = []#[0.001,0.005,0.01,0.05,0.10]
            intersection_methods = methods[:]

            if len(ignore_methods_for_intersection_matrix)>0:
                print 'Ignoring methods from intersection matrix: %s' % ignore_methods_for_intersection_matrix
                for m in ignore_methods_for_intersection_matrix:
                      for method in intersection_methods[:]:
                          if (m in method) and (method in intersection_methods):
                              intersection_methods.remove(method)

            if len(intersection_methods)==0:
                print 'No methods for comparison'
            elif include_intersection_matrix and len(intersection_sizes)>0:
                report.addSection('Pairwise intersections',0,clearPage=True)

                for k in intersection_sizes:
                    report.addSection('Top %s' % (100 * k) + r"\%",1)
                    print 'k=%s' % k

                    intersection_matrix = np.eye(len(intersection_methods))
                    i_pair = 1

                    for i in range(0,len(intersection_methods)-1):
                        method1 = intersection_methods[i]
                        pairs1 = pca_utils.get_top_k_pairs_from_distance_file((distances_dir if analysis_type == 'model' else analysis_dir) + '%s_distances_%s.txt' % (method1, distance_measure),k)

                        for j in range(i+1,len(intersection_methods)):
                            method2 = intersection_methods[j]
                            pairs2 = pca_utils.get_top_k_pairs_from_distance_file((distances_dir if analysis_type == 'model' else analysis_dir) + '%s_distances_%s.txt' % (method2,distance_measure),k)

                            #compute Jaccard similarity
                            similarity = 1.0 * len(utils.get_intersection(pairs1,pairs2))
                            similarity /= (len(pairs1) + len(pairs2) - similarity)

                            intersection_matrix[i,j] = intersection_matrix[j,i] = similarity

                            sys.stdout.write('\r%.2f%%...' % (100.0 * i_pair / ((len(intersection_methods) * (len(intersection_methods)-1) / 2.0))))
                            sys.stdout.flush()
                            i_pair += 1

                    #intersection_filename = 'intersections_%s.txt' % (str(k).replace('.',''))
                    #np.savetxt(distances_dir + intersection_filename,intersection_matrix,delimiter='\t')

                    report.addGroupFigure('Intersections of top %s' % (100 * k) + r"\%" + ' of sorted pairs',
                          [draw_utils.draw_heatmap(intersection_matrix,distances_dir + 'plots/intersections/','intersections_%s' % (str(k).replace('.','')), format = 'eps')],
                          image_caption_list = ['Jaccard similarity'],
                          image_angle=270,
                          column_count=1
                          )

            report.compile(distances_dir + 'report/', report_name)
