import sys, getopt, os, subprocess, re, math
import numpy as np
import scipy.stats
from LatexBuilder import LatexBuilder
from utils import *
from gdv_utils import *
import settings

inputTypeSubDir = 'counts/'
outputTypeSubDir = 'counts_comparison/'
countingExecutable = '../count_tmp_graphlets'

gnuplotTemplate = file(settings.GNUPLOT_SCATTER_TEMPLATE_FILE, 'r').read()
gnuplotHeatmapTemplate = file(settings.GNUPLOT_HEATMAP_TEMPLATE_FILE, 'r').read()


gdvs = dict() #dictionary node_pair -> (method -> gdvs)
methodsMinValues = dict()
methodsMaxValues = dict()


def getMethodsComparisonPlot(plotsDirectory, networkName, method1,method2,png = True):
    figExt = 'png' if png else 'eps'
    if os.path.exists(plotsDirectory + figExt + '/' + networkName  + '-%s-vs-%s.%s' % (method1, method2, figExt)):
        return networkName  + '-%s-vs-%s' % (method1, method2)
    elif os.path.exists(plotsDirectory + figExt + '/' + networkName  + '-%s-vs-%s.%s' % (method2, method1, figExt)):
        return networkName  + '-%s-vs-%s' % (method2, method1)
    else:
        print 'Creating plot comparing %s and %s' % (method1,method2)

        #set template values
        template_values = {
            'title': networkName.replace('_','-'),
            'x_label': getMethodName(method1),
            'y_label': getMethodName(method2),
            'input_file': '../'  + networkName + '-comparison.txt',
            'output_file': './eps/'  + networkName + '-%s-vs-%s' % (method1, method2),
            'x_min': math.floor(10*(methodsMinValues[method1] if methodsMinValues.has_key(method1) and methodsMinValues[method1]<1 else 0))/10,
            'x_max': math.ceil(10*(methodsMaxValues[method1] if methodsMaxValues.has_key(method1) and methodsMaxValues[method1]>0 else 1))/10,
            'y_min': math.floor(10*(methodsMinValues[method2] if methodsMinValues.has_key(method1) and methodsMinValues[method2]<1 else 0))/10,
            'y_max': math.ceil(10*(methodsMaxValues[method2] if methodsMaxValues.has_key(method2) and methodsMaxValues[method2]>0 else 1))/10,
            'x_col': methods.index(method1)+2,
            'y_col': methods.index(method2)+2
        }

        gnuplotFile = networkName + '-%s-vs-%s.gnuplot' % (method1, method2)

        #write gnuplot file
        open(plotsDirectory + gnuplotFile, mode='wb').write(gnuplotTemplate % template_values)

        #run gnuplot
        p = subprocess.Popen(['gnuplot', gnuplotFile], stdout=subprocess.PIPE, cwd = plotsDirectory)
        p_output = p.stdout.read()
        if p_output != '':
            print(p_output)


        #convert eps to png
        if png:
            print 'Converting to png...'
            p = subprocess.Popen(
                ['convert','-density', '300',
                 './eps/'  + networkName + '-%s-vs-%s.eps' % (method1, method2),
                 './png/'  + networkName + '-%s-vs-%s.png' % (method1, method2)],
                stdout=subprocess.PIPE, cwd = plotsDirectory)
            p_output = p.stdout.read()
            if p_output != '':
                print(p_output)

        return plotsDirectory + figExt + '/' + networkName  + '-%s-vs-%s.%s' % (method1, method2, figExt)



def readGdvs(inputDir, networkName, methodName):


    gdvsFile = inputDir

    if 'sgdvs' in methodName:
        gdvsFile += 'static/'
    elif 'stgdvs' in methodName:
        gdvsFile += 'static-temporal/'
        if '_stgdvs' in methodName:
            gdvsFile += methodName[:methodName.find('_stgdvs')] + '/'

    gdvsFile += networkName + '_' + methodName + '.txt'

    sF=open(gdvsFile,'r')
    lines=sF.readlines()

    print 'Loading %s (%s lines)...' % (gdvsFile, len(lines))

    methodsMinValues[methodName] = 1
    methodsMaxValues[methodName] = 0

    node_dict=dict()
    s_firstLine=lines.pop(0)
    s_nodeNames=s_firstLine.split(' ')


    row=1
    for line in lines:
        sys.stdout.write('\rrow %s/%s...' % (row,len(lines)))
        sys.stdout.flush()

        lineArr=line.split()
        ##create keys (node1-node2) for dict
        node1=lineArr[0].strip('\n')
        column=0
        #print 'node1-'+node1
        for node2 in s_nodeNames[row+3:]:
          node2 = node2.strip('\n')
          ikey=(min(node1,node2),max(node1,node2))
          ivalue=float(lineArr[column+2+row])

          if math.isnan(ivalue):
              ivalue = 0

          if gdvs.has_key(ikey):
              gdvs[ikey][methodName] = ivalue
          else:
              gdvs[ikey] = {methodName: ivalue}

          if ivalue < methodsMinValues[methodName]:
              methodsMinValues[methodName] = ivalue

          if ivalue > methodsMaxValues[methodName]:
              methodsMaxValues[methodName] = ivalue

          #print 'key-'+ikey+' value-'+xyArr
          #print 'column-'+str(column)+' : key-'+ikey+' : value-'+ivalue
          column+=1
        row+=1
    sF.close()



def createHeatmap(plotsDirectory, networkName, methodList, methods, correlationType, nameSuffix = '', convertFiguresToPng = True, output_suffix = ''):

    #if not creating heatmap for all methods, filter the matrix and save into separate file
    if methodList == methods:
        matrixFile = '../'  + networkName + '-correlations-%s-matrix%s.txt' % (correlationType, output_suffix)
    else:
        matrix = np.loadtxt(os.path.dirname(os.path.dirname(plotsDirectory))+ '/' +  networkName + '-correlations-%s-matrix%s.txt' % (correlationType,output_suffix))
        indicesToKeep = [methods.index(m)  for m in methodList]
        methods = methodList
        matrix = matrix[indicesToKeep,:][:,indicesToKeep]

        np.savetxt(os.path.dirname(os.path.dirname(plotsDirectory))+ '/' +  networkName + '-correlations-%s-matrix-filtered-%s.txt' % (correlationType,nameSuffix), matrix, delimiter='\t')
        matrixFile = '../'  + networkName + '-correlations-%s-matrix-filtered-%s.txt' % (correlationType,nameSuffix)


    gnuplotHeatMapFile = networkName + '-correlations-%s%s.gnuplot' % (correlationType, '-'+nameSuffix if nameSuffix !='' else '')
    template_values = {
        'title': 'Pairwise %s correlations for graphlet counting in %s' % (correlationType.capitalize(), networkName.replace('_','-')),
        'input_file': matrixFile,
        'output_file': './eps/'  + networkName + '-correlations-%s%s'  % (correlationType, '-'+nameSuffix if nameSuffix !='' else ''),
        'row_count': len(methodList),
        'row_descriptions': ','.join(['"%s" %s' % (methods[i].replace('_','-').replace('snapshots','sn').replace('aggregated','ag'),i)  for i in range(0,len(methods)) if methods[i] in methodList])
    }
    open(plotsDirectory + gnuplotHeatMapFile, mode='wb').write(gnuplotHeatmapTemplate % template_values)

    p = subprocess.Popen(['gnuplot', gnuplotHeatMapFile], stdout=subprocess.PIPE, cwd = plotsDirectory)
    print p.stdout.read()

    if convertFiguresToPng:
        p = subprocess.Popen(['convert','-density', '300',
                     './eps/'  + networkName + '-correlations-%s%s.eps'  % (correlationType, '-'+nameSuffix if nameSuffix !='' else ''),
                     './png/'  + networkName + '-correlations-%s%s.png'  % (correlationType, '-'+nameSuffix if nameSuffix !='' else '')],
                             stdout=subprocess.PIPE, cwd = plotsDirectory)

    return gnuplotHeatMapFile.replace('.gnuplot','')


inputSubDir = ''
convertFiguresToPng = False
reportName = ''
ignore_methods = []
run_methods = []
generateReport = True
createFigures = True
generateComparisonFiles = True
output_suffix = ''
methods = None

opts, args = getopt.getopt(sys.argv[1:], "s:gph", ["subdir=","ignore-methods=", "run-methods=", "help","no-report","report-name=", "use-methods="])
for o, a in opts:
    if o in ('-s', '--subdir'):
        inputSubDir = a  + '/'
    elif o in ('-p'):
        convertFiguresToPng = True
    elif o in ('--no-report'):
        generateReport = False
    elif o in ('-g'):
        generateComparisonFiles = False
    elif o in ('--use-methods'):
        methods = a.split(',')
    elif o in ('--ignore-methods'):
        ignore_methods = a.split(',')
    elif o in ('--report-name'):
        reportName = a
        output_suffix = '-report-' + a
    elif o in ('--run-methods'):
        if a == 'standard':
            a =  'aggregated_1_sgdvs_3, aggregated_1_sgdvs_4, aggregated_1_sgdvs_5, stgdvs_3, stgdvs_4, stgdvs_5, dgdvs_3_3_1, dgdvs_4_3_1, dgdvs_5_3_1, dgdvs_6_3_1, dgdvs_7_3_1, dgdvs_6_4_1, dcgdvs_3_3_1, dcgdvs_4_3_1, dcgdvs_5_3_1, dcgdvs_6_3_1, dcgdvs_7_3_1, dcgdvs_6_4_1'
        run_methods = a.split(',')
    elif o in ('-h','--help'):
        print """
Generates report comparing various node similarity measures
Usage:
%s network_name
-s,--subdir      Input/output subdirectory
-p               Convert eps figures to png format (and use them for the report)
-g               Whether not to generate comparison files
--no-report      Do not generate report
--ignore-methods Methods to ignore (if method name contains any of the input substrings, ignore it)
--use-methods    Methods to use (otherwise use all)
--run-methods    Methods to run (using static or dynamic run script)
--output-suffix  Output suffix for comparison files
-h,--help        Print this""" % sys.argv[0]
        sys.exit(0)

networkName = args[0]
inputDir = settings.BASE_OUTPUT_DIR + inputSubDir + networkName + '/' + inputTypeSubDir

reportName = 'report_%s%s' % (networkName, '' if reportName == '' else '_'+reportName)


print 'Comparing gdvs for %s network...' % networkName

if len(run_methods)>0:
    print 'Running %s methods: %s' % (len(run_methods), run_methods)
    i_count = 1
    for m in run_methods:

        print '%s/%s: %s...' % (i_count, len(run_methods), m)
        i_count += 1
        m_dict = getMethodDetailsDict(m)
        run_params = ['python',
             './src/run_%s_counting.py' % ('dynamic' if ('d' in m_dict['type']) else 'static'),
             '-s'  if inputSubDir != '' else '',
             inputSubDir if inputSubDir != '' else '']


        if 'd' in m_dict['type']:
            run_params += ['-d','tab']
            run_params.append(networkName)
            run_params.append(m)
        else:
            run_params.append(networkName)
            run_params.append(m)
            if not os.path.exists('../input/%s%s/leda_format/' % (inputSubDir, networkName)):
                print 'Converting to leda...'
                p = subprocess.Popen(['python','./src/convert_network_format.py',
                                      '-s' if inputSubDir != '' else '',
                                      inputSubDir[:-1] if inputSubDir != '' else '',
                                      networkName,
                                      'list',
                                      'leda'], stdout=subprocess.PIPE)
                for line in iter(p.stdout.readline, ''):
                    sys.stdout.write(line)
                p.stdout.close()

        p = subprocess.Popen(run_params, stdout=subprocess.PIPE)
        for line in iter(p.stdout.readline, ''):
            sys.stdout.write(line)
        p.stdout.close()


if methods is None:
    methods = getGdvsMethods(networkName, inputSubDir)

print 'Ignoring methods: %s' % ignore_methods
for m in ignore_methods:
      for method in methods[:]:
          if (m in method) and (method in methods):
              methods.remove(method)



methodsDetails =[getMethodDetailsDict(m) for m in methods]


print 'Comparing the following %s methods : %s' % (len(methods),', '.join(methods))


if not os.path.exists(settings.BASE_OUTPUT_DIR + inputSubDir + networkName + '/' + outputTypeSubDir):
    os.makedirs(settings.BASE_OUTPUT_DIR + inputSubDir + networkName + '/' + outputTypeSubDir)

if not os.path.exists(settings.BASE_OUTPUT_DIR + inputSubDir + networkName + '/' + outputTypeSubDir + 'plots/'):
    os.makedirs(settings.BASE_OUTPUT_DIR + inputSubDir + networkName + '/' + outputTypeSubDir + 'plots/')
if not os.path.exists(settings.BASE_OUTPUT_DIR + inputSubDir + networkName + '/' + outputTypeSubDir + 'plots/eps/'):
    os.makedirs(settings.BASE_OUTPUT_DIR  + inputSubDir+ networkName + '/' + outputTypeSubDir + 'plots/eps/')
if not os.path.exists(settings.BASE_OUTPUT_DIR + inputSubDir + networkName + '/' + outputTypeSubDir + 'plots/png/'):
    os.makedirs(settings.BASE_OUTPUT_DIR  + inputSubDir+ networkName + '/' + outputTypeSubDir + 'plots/png/')




plotsDirectory = settings.BASE_OUTPUT_DIR + inputSubDir + networkName + '/' + outputTypeSubDir + 'plots/'
reportsDirectory = settings.BASE_OUTPUT_DIR + inputSubDir + networkName + '/' + outputTypeSubDir + 'reports/'

if generateComparisonFiles:
    print 'Reading gdv files...'
    i=1
    for method in methods:
        print '\n%s/%s %s...' % (i,len(methods), method)
        #sys.stdout.write('\r%s/%s %s...' % (i,len(methods), method))
        #sys.stdout.flush()
        i+=1
        readGdvs(inputDir,networkName,method)


    i_count = 1
    i_max = len(methods) * (len(methods)-1) / 2
    node_pairs = sorted(gdvs.keys())

    print '\nCreating auxiliary files...'
    correlationsFile = open(settings.BASE_OUTPUT_DIR + inputSubDir + networkName + '/' + outputTypeSubDir + networkName + '-correlations.txt',mode='wb')
    correlationsFile.write('Method1\tMethod2\tPearson\tSpearman\n')

    methodsAllValues = dict()
    for method in methods:
        methodsAllValues[method] = []


    print 'Combining all values...'

    i=1
    with open(settings.BASE_OUTPUT_DIR + inputSubDir + networkName + '/' + outputTypeSubDir + networkName + '-comparison%s.txt' % output_suffix, mode='wb') as outputFile:
        outputFile.write('node_pair\t%s\n' % '\t'.join(methods))
        for k in node_pairs:
            sys.stdout.write('\r%s/%s...' % (i,len(node_pairs)))
            sys.stdout.flush()
            i+=1
            nodePairVales = []
            for method in methods:
                if not gdvs[k].has_key(method):
                    gdvs[k][method] = -1
                nodePairVales.append('%f' % gdvs[k][method])
                methodsAllValues[method].append(gdvs[k][method])
            outputFile.write('%s\t%s\n' % (k[0] + '-' + k[1], '\t'.join(nodePairVales)))


    #for heatmap
    correlationsPearsonMatrixFile = open(settings.BASE_OUTPUT_DIR + inputSubDir + networkName + '/' + outputTypeSubDir + networkName + '-correlations-pearson-matrix%s.txt' % output_suffix, mode='wb')
    correlationsSpearmanMatrixFile = open(settings.BASE_OUTPUT_DIR + inputSubDir + networkName + '/' + outputTypeSubDir + networkName + '-correlations-spearman-matrix%s.txt' % output_suffix, mode='wb')

    print '\nComparing methods...'

    for i in range(0,len(methods)):
        method1 = methods[i]
        for j in range(0,len(methods)):
            method2 = methods[j]

            values1 = []
            values2 = []

            #consider only those node pair for which there are similarities computed by both methods
            for ii in range(len(node_pairs)):
                if methodsAllValues[method1][ii] != -1 and methodsAllValues[method2] != -1:
                    values1.append(methodsAllValues[method1][ii])
                    values2.append(methodsAllValues[method2][ii])


            #compute correlations
            pearsonCorr = np.corrcoef(values1,values2)[0,1]
            spearmanCorr = scipy.stats.spearmanr(values1,values2)[0]

            #need complete matrix
            correlationsPearsonMatrixFile.write('%s\t' % pearsonCorr)
            correlationsSpearmanMatrixFile.write('%s\t' % spearmanCorr)

            if i <= j:
                continue


            sys.stdout.write('\r%s/%s. %s vs %s' % (i_count,i_max,getMethodName(method1),getMethodName(method2)))
            sys.stdout.flush()
            i_count+=1


            correlationsFile.write('%s\t%s\t%s\t%s\n' % (getMethodName(method1),getMethodName(method2),pearsonCorr,spearmanCorr))

        correlationsPearsonMatrixFile.write('\n')
        correlationsSpearmanMatrixFile.write('\n')

    correlationsSpearmanMatrixFile.close()
    correlationsPearsonMatrixFile.close()






if generateReport:
    #group plots so that to answer different questions:
    #1. Effect of network construction parameters (fixed method - varied network construction)
    #2. Static vs dynamic (s vs st vs d vs dc)
    #3. Effect of graphlet size (vary n)
    #4. Effect of number of events (vary k)


    #generate latex summary
    print 'Generating summary report...'

    report = LatexBuilder(
        author = 'Yuriy Hulovatyy',
        title = 'Methods comparison for \'%s\' network' %  networkName.upper().replace('_','-'),
        graphicspath= '../../plots/%s/' % ('png' if convertFiguresToPng else 'eps')
        )

    #------------------------------------------------------------------------------------------------------
    print 'Part 1: The effect of network construction parameters...'
    methods_temp_dict = {}
    any_relevant_methods = False

    for m in methodsDetails:
        mKey = '%sgdvs' % m['type']
        if not 's' in m['type']:
            mKey += '_%s_%s_%s' % (m['k'], m['n'], m['delta_t'])
        else:
            mKey += '_%s' % m['n']

        if methods_temp_dict.has_key(mKey):
            methods_temp_dict[mKey].append(m['network_construction'])
            any_relevant_methods = True
        else:
            methods_temp_dict[mKey] = [m['network_construction']]

    if not any_relevant_methods:
        print 'No methods with the same type and parameters but different network construction'
    else:
        report.addSection('Network construction parameters',0, clearPage=True)

        mKeyList = sorted(methods_temp_dict.keys(), cmp = compareMethodNames)

        for mKey in mKeyList:
            if len(methods_temp_dict[mKey])<2:
                continue

            report.addSection(getMethodName(mKey, onlyType=True),1)
            image_caption_list = []
            plotList = []

            for n1 in methods_temp_dict[mKey]:
                for n2 in methods_temp_dict[mKey]:
                    if n1<=n2:
                        continue

                    plotList.append(getMethodsComparisonPlot(plotsDirectory, networkName, n1 + '_' + mKey,n2 + '_' + mKey, convertFiguresToPng))
                    image_caption_list.append('%s vs %s' % (n1.replace('_','-'),n2.replace('_','-')))

            report.addGroupFigure('Comparison for %s' % getMethodName(mKey),
                          plotList,
                          image_caption_list = image_caption_list,
                          image_angle=270,
                          column_count=3
                          )
            if len(methods_temp_dict[mKey])>1:
                report.addGroupFigure('Correlations %s' % mKey.replace('_','-'),
                                      [createHeatmap(plotsDirectory, networkName, [nM + '_' + mKey for nM in methods_temp_dict[mKey]],methods,'pearson','net-constr-%s' % mKey, convertFiguresToPng),
                                       createHeatmap(plotsDirectory, networkName, [nM + '_' + mKey for nM in methods_temp_dict[mKey]],methods,'spearman','net-constr-%s' % mKey, convertFiguresToPng)],
                                      image_caption_list= ['Pearson', 'Spearman'],
                                      image_angle=270
                                      )




    #------------------------------------------------------------------------------------------------------

    #for the next report parts, in case there are multiple construction params, choose only one
    default_aggregated_params = 'aggregated_1'
    default_snapshot_params = 'snapshots_1month_1'

    #correspondence in sizes between static and dynamic graphlets
    grahplet_size_correspondence_dict = {
        '3_3': 3,
        '6_4': 4,
        '10_5': 5,
    }

    methods_temp_dict = {
        3: {},
        4: {},
        5: {}
    }


    any_relevant_methods = False

    for m in methodsDetails:
        mKey = '%sgdvs' % m['type']
        size_key = None
        if not 's' in m['type']:
            mKey += '_%s_%s_%s' % (m['k'], m['n'], m['delta_t'])
            if grahplet_size_correspondence_dict.has_key('%s_%s' % (m['k'], m['n'])):
                size_key =  grahplet_size_correspondence_dict['%s_%s' % (m['k'], m['n'])]
        else:
            mKey += '_%s' % m['n']
            if methods_temp_dict.has_key(m['n']):
                size_key = m['n']

        if size_key is None:
            continue
        elif not methods_temp_dict.has_key(size_key):
            methods_temp_dict[size_key] = {}

        if methods_temp_dict[size_key].has_key(mKey):
            methods_temp_dict[size_key][mKey].append(m['network_construction'])
        else:
            methods_temp_dict[size_key][mKey] = [m['network_construction']]

        if len(methods_temp_dict[size_key].keys())>1:
            any_relevant_methods = True

    #------------------------------------------------------------------------------------------------------
    print 'Part 2: Static vs dynamic...'



    if not any_relevant_methods:
        print 'No methods to compare for static vs dynamic'
    else:
        methodsForHeatmap = []
        report.addSection('Static vs dynamic',0, clearPage=True)
        #remove redundant network constructions
        for size_key in methods_temp_dict.keys():

            for mKey in methods_temp_dict[size_key]:
                if len(methods_temp_dict[size_key][mKey])>1:
                    if 'st' in mKey or 'dgdvs' in mKey or 'dcgdvs' in mKey or 'dzgdvs' in mKey or 'dczgdvs' in mKey:
                        methods_temp_dict[size_key][mKey] = [default_snapshot_params]
                    else:
                        methods_temp_dict[size_key][mKey] = [default_aggregated_params]

            if len(methods_temp_dict[size_key])<2:
                continue

            report.addSection('%s-graphlets' % size_key,1)

            method_names_sorted = sorted(methods_temp_dict[size_key], cmp = compareMethodNames)

            #methodsForHeatmap += [((methods_temp_dict[size_key][m1][0] + '_') if methods_temp_dict[size_key][m1][0] != '' else '')+  m1 for m1 in method_names_sorted]

            image_caption_list = []
            plotList = []
            heatmapMethodList = set()

            #static vs all
            for m1 in method_names_sorted:
                if not 'sgdvs' in m1:
                    continue
                for m2 in method_names_sorted:
                    if m1==m2:
                        continue

                    #print '%s: comparing %s and %s' % (size_key,m1,m2)

                    if methods_temp_dict[size_key][m1][0] != '':
                        m1_full = methods_temp_dict[size_key][m1][0] + '_' +  m1
                    else:#in case there is no special network construction info
                        m1_full = m1
                    if methods_temp_dict[size_key][m2][0] != '':
                        m2_full = methods_temp_dict[size_key][m2][0] + '_' +  m2
                    else:
                        m2_full = m2

                    heatmapMethodList.add(m1_full)
                    heatmapMethodList.add(m2_full)

                    plotList.append(getMethodsComparisonPlot(plotsDirectory, networkName, m1_full, m2_full,convertFiguresToPng))
                    image_caption_list.append('%s vs %s' % (getMethodName(m1),getMethodName(m2)))
                if len(plotList)>0:
                    report.addGroupFigure('Static vs all for graphlets of size %s' % size_key,
                                  plotList,
                                  image_caption_list = image_caption_list,
                                  image_angle=270,
                                  column_count=3
                                  )

                if len(plotList)>1:
                    heatmapMethodList = sorted(list(heatmapMethodList), cmp = compareMethodNames)
                    report.addGroupFigure('Correlations',
                                          [createHeatmap(plotsDirectory, networkName, heatmapMethodList, methods,'pearson','s-vs-d-%s' % size_key, convertFiguresToPng),
                                           createHeatmap(plotsDirectory, networkName, heatmapMethodList, methods,'spearman','s-vs-d-%s' % size_key, convertFiguresToPng)],
                                          image_caption_list= ['Pearson', 'Spearman'],
                                          image_angle=270
                                          )

            image_caption_list = []
            plotList = []
            heatmapMethodList = set()


            #st vs d vs dc
            for m1 in method_names_sorted:
                if 'sgdvs' in m1:
                    continue
                for m2 in method_names_sorted:
                    if 'sgdvs' in m2:
                        continue
                    if m1<=m2:
                        continue

                    if methods_temp_dict[size_key][m1][0] != '':
                        m1_full = methods_temp_dict[size_key][m1][0] + '_' +  m1
                    else:#in case there is no special network construction info
                        m1_full = m1
                    if methods_temp_dict[size_key][m2][0] != '':
                        m2_full = methods_temp_dict[size_key][m2][0] + '_' +  m2
                    else:
                        m2_full = m2

                    heatmapMethodList.add(m1_full)
                    heatmapMethodList.add(m2_full)

                    plotList.append(getMethodsComparisonPlot(plotsDirectory, networkName,m1_full,m2_full,convertFiguresToPng))
                    image_caption_list.append('%s vs %s' % (getMethodName(m1),getMethodName(m2)))
            if len(plotList)>0:
                report.addGroupFigure('Static-temporal vs dynamic for graphlets of size %s' % size_key,
                              plotList,
                              image_caption_list = image_caption_list,
                              image_angle=270,
                              column_count=3
                              )

            if len(plotList)>1:
                heatmapMethodList = sorted(list(heatmapMethodList), cmp = compareMethodNames)
                report.addGroupFigure('Correlations',
                                      [createHeatmap(plotsDirectory, networkName, heatmapMethodList, methods,'pearson','st-vs-d-%s' % size_key, convertFiguresToPng),
                                       createHeatmap(plotsDirectory, networkName, heatmapMethodList, methods,'spearman','st-vs-d-%s' % size_key, convertFiguresToPng)],
                                      image_caption_list= ['Pearson', 'Spearman'],
                                      image_angle=270
                                      )

    #------------------------------------------------------------------------------------------------------
    print 'Part 3: Graphlet size...'
    methods_by_type = {}
    any_relevant_methods = False

    for m_type in getAllMethodTypes():
        methods_by_type[m_type] = []
        for g_size in methods_temp_dict.keys():

            methods_by_type[m_type] += [((methods_temp_dict[g_size][m][0] + '_') if methods_temp_dict[g_size][m][0] != '' else '') + m for m in methods_temp_dict[g_size] if ((m_type+'gdvs') in m)]
        if len(methods_by_type[m_type])>1:
            any_relevant_methods = True


    if not any_relevant_methods:
        print 'No methods to compare for the effect of graphlet size'
    else:
        report.addSection('Graphlet number of nodes',0, clearPage=True)

        for m_type in sorted(methods_by_type.keys(), cmp = compareMethodTypes):

            image_caption_list = []
            plotList = []
            methods_to_compare = methods_by_type[m_type]

            if len(methods_to_compare)>1:
                report.addSection(getMethodTypeDescr(m_type),1)

            for m1 in methods_to_compare:
                for m2 in methods_to_compare:
                    if m1<=m2:
                        continue

                    #print 'comparing %s and %s' % (m1,m2)
                    plotList.append(getMethodsComparisonPlot(plotsDirectory, networkName,m1,m2,convertFiguresToPng))
                    image_caption_list.append('%s vs %s' % (getMethodName(m1),getMethodName(m2)))

            if len(plotList)>0:
                report.addGroupFigure('Graphlet size for %s' % getMethodTypeDescr(m_type),
                              plotList,
                              image_caption_list = image_caption_list,
                              image_angle=270,
                              column_count=3
                              )

            if len(plotList)>1:
                report.addGroupFigure('Correlations',
                                      [createHeatmap(plotsDirectory, networkName, methods_to_compare, methods,'pearson','g-size-%s' % m_type, convertFiguresToPng),
                                       createHeatmap(plotsDirectory, networkName, methods_to_compare, methods,'spearman','g-size-%s' % m_type, convertFiguresToPng)],
                                      image_caption_list= ['Pearson', 'Spearman'],
                                      image_angle=270
                                      )

    #------------------------------------------------------------------------------------------------------
    print 'Part 4: Number of events...'
    methods_by_type = {}
    any_relevant_methods = False


    methods_temp_dict = {}
    for m in methodsDetails:
        if 's' in  m['type']:
            continue
        mKey = '%s_%s' % (m['n'], m['delta_t'])

        if not methods_temp_dict.has_key(m['type']):
            methods_temp_dict[m['type']] = {}

        if methods_temp_dict[m['type']].has_key(mKey):
            methods_temp_dict[m['type']][mKey].append(m['full_name'])
            any_relevant_methods = True
        else:
            methods_temp_dict[m['type']][mKey] = [m['full_name']]

    #choose one network construction (if there are more than one)
    for m_type in methods_temp_dict:
        for mKey in methods_temp_dict[m_type]:
            constr_parms = set()

            for m in methods_temp_dict[m_type][mKey]:
                m_temp  = getMethodDetailsDict(m)
                constr_parms.add(m_temp['network_construction'])
            if len(constr_parms)>1:
                m_list_temp  = list(methods_temp_dict[m_type][mKey])
                for m in m_list_temp:
                    m_temp = getMethodDetailsDict(m)
                    if default_snapshot_params != m_temp['network_construction']:
                        methods_temp_dict[m_type][mKey].remove(m)

    if not any_relevant_methods:
        print 'No methods to compare for the effect of graphlet size'
    else:
        report.addSection('Graphlet number of events',0, clearPage=True)
        for m_type in sorted(methods_temp_dict.keys(), cmp = compareMethodTypes):

            if len(methods_temp_dict[m_type]) == 0:
                continue

            has_various_event_counts = False
            for mKey in methods_temp_dict[m_type]:
                if len(methods_temp_dict[m_type][mKey])>=2:
                    has_various_event_counts = True
            if not has_various_event_counts:
                continue

            report.addSection(getMethodTypeDescr(m_type),1)
            for mKey in methods_temp_dict[m_type]:
                if len(methods_temp_dict[m_type][mKey])<2:
                    continue

                image_caption_list = []
                plotList = []
                temp_methods_list = sorted(methods_temp_dict[m_type][mKey], cmp = compareMethodNames)

                #print 'methods_temp_dict[%s][%s] = %s' % (m_type,mKey,methods_temp_dict[m_type][mKey])
                for n1 in temp_methods_list:
                    for n2 in temp_methods_list:
                        if n1<=n2:
                            continue

                        plotList.append(getMethodsComparisonPlot(plotsDirectory, networkName, n1,n2,convertFiguresToPng))
                        image_caption_list.append('%s vs %s' % (getMethodName(n1),getMethodName(n2)))

                report.addGroupFigure('Comparison for %s (n=%s)' % (getMethodTypeDescr(m_type),mKey.split('_')[0]),
                              plotList,
                              image_caption_list = image_caption_list,
                              image_angle=270
                              )
                if len(temp_methods_list)>1:
                    report.addGroupFigure('Correlations for %s (n=%s)' % (getMethodTypeDescr(m_type),mKey.split('_')[0]),
                                          [createHeatmap(plotsDirectory, networkName, temp_methods_list,methods,'pearson','k-size-%s-%s' % (m_type,mKey), convertFiguresToPng),
                                           createHeatmap(plotsDirectory, networkName, temp_methods_list,methods,'spearman','k-size-%s-%s' % (m_type,mKey), convertFiguresToPng)],
                                          image_caption_list= ['Pearson', 'Spearman'],
                                          image_angle=270
                                          )


    #---------------------------------------------------
    if len(methods)>2:
        report.addSection('All vs All',0)
        report.addGroupFigure('Correlations',
                              [createHeatmap(plotsDirectory, networkName, methods,methods,'pearson','', convertFiguresToPng),
                               createHeatmap(plotsDirectory, networkName, methods,methods,'spearman','', convertFiguresToPng)],
                              image_caption_list= ['Pearson', 'Spearman'],
                              image_angle=270
                              )


    report.compile(reportsDirectory, reportName, convertFiguresToPng)




