def corpusMelodicIntervalSearch(show = True):
    # this version compares china to netherlands
    from music21 import corpus
    from music21.analysis import discrete

    mid = discrete.MelodicIntervalDiversity()
    groupEast = corpus.search('shanxi', 'locale')
    groupWest = corpus.search('niederlande', 'locale') 

    msg = []
    for name, group in [('shanxi', groupEast), ('niederlande', groupWest)]:
        intervalDict = {}
        workCount = 0
        intervalCount = 0
        seventhCount = 0
        for fp, n in group:
            workCount += 1
            s = converter.parse(fp, number=n)
            intervalDict = mid.countMelodicIntervals(s, found=intervalDict)
        
        for key in sorted(intervalDict.keys()):
            intervalCount += intervalDict[key][1] # second value is count
            if key in ['m7', 'M7']:
                seventhCount += intervalDict[key][1] 

        pcentSevenths = round(((seventhCount / float(intervalCount)) * 100), 4)
        msg.append('locale: %s: found %s percent melodic sevenths, out of %s intervals in %s works' % (name, pcentSevenths, intervalCount, workCount))
#         for key in sorted(intervalDict.keys()):
#             print intervalDict[key]

    for sub in msg: 
        if show == True:
            print sub

def corpusMelodicIntervalSearchBrief(show=False):
    # try for the most concise representation
    from music21 import corpus, analysis
    mid = analysis.discrete.MelodicIntervalDiversity()
    msg = []
    for region in ['shanxi', 'fujian']:
        intervalDict = {}
        workCount = 0
        intervalCount = 0
        seventhCount = 0
        for fp, n in corpus.search(region, 'locale'):
            workCount += 1
            s = converter.parse(fp, number=n)
            intervalDict = mid.countMelodicIntervals(s, found=intervalDict)
        for key in intervalDict.keys():
            intervalCount += intervalDict[key][1] # second value is count
            if key in ['m7', 'M7']:
                seventhCount += intervalDict[key][1]


        pcentSevenths = round((seventhCount / float(intervalCount) * 100), 4)

        msg.append('locale: %s: found %s percent melodic sevenths, out of %s intervals in %s works' % (region, pcentSevenths, intervalCount, workCount))

    if show == True:
        for sub in msg: 
            print sub

 def testSearch12(self):
     # searching virtual entries
     searchResults = corpus.search('coltrane', field='composer')
     self.assertEqual(len(searchResults) > 0, True)
     # returns items in pairs: url and work number
     self.assertEqual(searchResults[0].sourcePath,
         'http://impromastering.com/uploads/transcription_file/file/196/Giant_Steps__John_Coltrane_C.xml')

def corpusMelodicIntervalSearchBrief(show=False):
    # try for the most concise representation
    from music21 import corpus, analysis
    melodicIntervalDiversity = analysis.discrete.MelodicIntervalDiversity()
    message = []
    for region in ['shanxi', 'fujian']:
        intervalDict = {}
        workCount = 0
        intervalCount = 0
        seventhCount = 0
        for metadataEntry in corpus.search(region, field='locale'):
            workCount += 1
            score = corpus.parse(
                metadataEntry.sourcePath,
                number=metadataEntry.number,
                )
            intervalDict = melodicIntervalDiversity.countMelodicIntervals(
                score, found=intervalDict)
        for key in intervalDict.keys():
            intervalCount += intervalDict[key][1] # second value is count
            if key in ['m7', 'M7']:
                seventhCount += intervalDict[key][1]
        pcentSevenths = round((seventhCount / float(intervalCount) * 100), 4)
        message.append('locale: %s: found %s percent melodic sevenths, out of %s intervals in %s works' % (region, pcentSevenths, intervalCount, workCount))
    if show == True:
        for sub in message: 
            print (sub)

 def testSearch03(self):
     searchResults = corpus.search('Taiwan', field='locale')
     self.assertEqual(len(searchResults), 27)
     pathInfo = sorted((str(searchResult.sourcePath), searchResult.number)
         for searchResult in searchResults)
     self.assertEqual(pathInfo, [
         ('essenFolksong/han1.abc', '269'),
         ('essenFolksong/han1.abc', '270'),
         ('essenFolksong/han1.abc', '271'),
         ('essenFolksong/han1.abc', '272'),
         ('essenFolksong/han1.abc', '273'),
         ('essenFolksong/han1.abc', '274'),
         ('essenFolksong/han1.abc', '335'),
         ('essenFolksong/han1.abc', '528'),
         ('essenFolksong/han1.abc', '529'),
         ('essenFolksong/han1.abc', '530'),
         ('essenFolksong/han2.abc', '204'),
         ('essenFolksong/han2.abc', '205'),
         ('essenFolksong/han2.abc', '206'),
         ('essenFolksong/han2.abc', '207'),
         ('essenFolksong/han2.abc', '208'),
         ('essenFolksong/han2.abc', '209'),
         ('essenFolksong/han2.abc', '210'),
         ('essenFolksong/han2.abc', '211'),
         ('essenFolksong/han2.abc', '212'),
         ('essenFolksong/han2.abc', '213'),
         ('essenFolksong/han2.abc', '214'),
         ('essenFolksong/han2.abc', '215'),
         ('essenFolksong/han2.abc', '216'),
         ('essenFolksong/han2.abc', '217'),
         ('essenFolksong/han2.abc', '218'),
         ('essenFolksong/han2.abc', '219'),
         ('essenFolksong/han2.abc', '220'),
         ])

    def xtestEx04(self):
        # what

        scSrc = scale.MajorScale()

        niederlande = corpus.search('niederlande', field='locale')

        results = {}
        for unused_name, group in [('niederlande', niederlande)]:
            workCount = 0

            for fp, n in group:
                workCount += 1
    
                s = converter.parse(fp, number=n)
    
                # derive a best-fit concrete major scale
                scFound = scSrc.derive(s)

                # if we find a scale with no unmatched pitches
                if len(scFound.match(s)['notMatched']) == 0:
                    # find out what pitches in major scale are not used
                    post = scFound.findMissing(s)
                    for p in post:
                        degree = scFound.getScaleDegreeFromPitch(p)
                        if degree not in results.keys():
                            results[degree] = 0
                        results[degree] += 1

        print ('Of %s works, the following major scale degrees are not used the following number of times:' % workCount)
        print (results)

def corpusFindMelodicSevenths(show = True):
    # find and display melodic sevenths
    import os
    from music21 import corpus
    from music21.analysis import discrete

    mid = discrete.MelodicIntervalDiversity()
    groupEast = corpus.search('shanxi', 'locale')
    groupWest = corpus.search('niederlande', 'locale') 

    found = []
    for name, group in [('shanxi', groupEast), ('niederlande', groupWest)]:
        for fp, n in group:
            s = converter.parse(fp, number=n)
            intervalDict = mid.countMelodicIntervals(s)
        
            for key in sorted(intervalDict.keys()):
                if key in ['m7', 'M7']:
                    found.append([fp, n, s])
                   
    results = stream.Stream()
    for fp, num, s in found: 
        environLocal.printDebug(['working with found', fp, num])
        # this assumes these are all monophonic
        noteStream = s.flat.getElementsByClass('Note')
        for i, n in enumerate(noteStream):
            if i <= len(noteStream) - 2:
                nNext = noteStream[i+1]
            else:
                nNext = None

            if nNext is not None:
                #environLocal.printDebug(['creating interval from notes:', n, nNext, i])
                i = interval.notesToInterval(n, nNext)
                environLocal.printDebug(['got interval', i.name])
                if i.name in ['m7', 'M7']:
                    #n.addLyric(s.metadata.title)
                    junk, fn = os.path.split(fp)
                    n.addLyric('%s: %s' % (fn, num))

                    m = noteStream.extractContext(n, 1, 2, forceOutputClass=stream.Measure)
                    m.makeAccidentals()
                    m.timeSignature = m.bestTimeSignature()
                    results.append(m)
    if show == True:
        results.show()

    def testExamplesC(self):

        from music21 import corpus, analysis, converter
        # Get an analysis tool
        mid = analysis.discrete.MelodicIntervalDiversity()
        results = []
        # Iterate over two regions
        for region in ['shanxi', 'fujian']:
            # Create storage units
            intervalDict = {}
            workCount = 0
            intervalCount = 0
            seventhCount = 0
            # Perform a location search on the corpus and iterate over 
            # resulting file name and work number
            for fp, n in corpus.search(region, 'locale'):
                workCount += 1
                # Parse the work and create a dictionary of intervals
                s = converter.parse(fp, number=n)
                intervalDict = mid.countMelodicIntervals(s, found=intervalDict)
            # Iterate through all intervals, and count totals and sevenths
            for label in intervalDict.keys():
                intervalCount += intervalDict[label][1] 
                if label in ['m7', 'M7']:
                    seventhCount += intervalDict[label][1]
            # Calculate a percentage and store results
            pcentSevenths = round((seventhCount / float(intervalCount) * 100), 
                            4)
            results.append((region, pcentSevenths, intervalCount, workCount))
    
        # Print results
        for region, pcentSevenths, intervalCount, workCount in results: 
            print('locale: %s: found %s percent melodic sevenths, out of %s intervals in %s works' % (region, pcentSevenths, intervalCount, workCount))

        region, pcentSevenths, intervalCount, workCount = results[0]
        self.assertEqual(region, 'shanxi')
        self.assertEqual(pcentSevenths, 3.1994)
        self.assertEqual(intervalCount, 4282)
        self.assertEqual(workCount, 77)

        region, pcentSevenths, intervalCount, workCount = results[1]
        self.assertEqual(region, 'fujian')
        self.assertEqual(pcentSevenths, 0.7654)
        self.assertEqual(intervalCount, 2613)
        self.assertEqual(workCount, 53)

 def testSearch03(self):
     searchResults = corpus.search("Taiwan", field="locale")
     self.assertEqual(len(searchResults), 27)
     pathInfo = sorted((searchResult.sourcePath, searchResult.number) for searchResult in searchResults)
     self.assertEqual(
         pathInfo,
         [
             (u"essenFolksong/han1.abc", u"269"),
             (u"essenFolksong/han1.abc", u"270"),
             (u"essenFolksong/han1.abc", u"271"),
             (u"essenFolksong/han1.abc", u"272"),
             (u"essenFolksong/han1.abc", u"273"),
             (u"essenFolksong/han1.abc", u"274"),
             (u"essenFolksong/han1.abc", u"335"),
             (u"essenFolksong/han1.abc", u"528"),
             (u"essenFolksong/han1.abc", u"529"),
             (u"essenFolksong/han1.abc", u"530"),
             (u"essenFolksong/han2.abc", u"204"),
             (u"essenFolksong/han2.abc", u"205"),
             (u"essenFolksong/han2.abc", u"206"),
             (u"essenFolksong/han2.abc", u"207"),
             (u"essenFolksong/han2.abc", u"208"),
             (u"essenFolksong/han2.abc", u"209"),
             (u"essenFolksong/han2.abc", u"210"),
             (u"essenFolksong/han2.abc", u"211"),
             (u"essenFolksong/han2.abc", u"212"),
             (u"essenFolksong/han2.abc", u"213"),
             (u"essenFolksong/han2.abc", u"214"),
             (u"essenFolksong/han2.abc", u"215"),
             (u"essenFolksong/han2.abc", u"216"),
             (u"essenFolksong/han2.abc", u"217"),
             (u"essenFolksong/han2.abc", u"218"),
             (u"essenFolksong/han2.abc", u"219"),
             (u"essenFolksong/han2.abc", u"220"),
         ],
     )

 def testSearch05(self):
     searchResults = corpus.search("bach")
     self.assertEqual(len(searchResults) > 120, True)

def _compute_PCs(out_fp):
    """Computes pitch class per measure principal components."""
    bachBundle = corpus.search('bwv')
    bachBundle = bachBundle.search('4/4')

    # NOTE: we should refactor this into a separate helper function: music21 Stream -> Pitch class histogram
    index =0
    data = {}
    for n in range(len(bachBundle)):
        data[n] = {}
        for i in range(30,100):
            data[n][i] = 0

    for n in range(len(bachBundle)):
        myPiece = bachBundle[n].parse()

        for m in myPiece.flat.getElementsByClass('Note'):
            data[n][m.midi] +=1

        print 'Number %i' % n

    new_data = np.array([data[0].values()]).astype(np.float64)
    new_data /= np.sum(new_data)

    for index in range(len(bachBundle)):
        temp = np.array([data[index].values()]).astype(np.float64)
        temp /= np.sum(temp)
        new_data =  np.concatenate((new_data,  temp)  , axis=0)

    print 'Statistics gathered!'
    save = new_data

###############################################################################
    bachBundle = corpus
    bachBundle = bachBundle.search('4/4')

    index =0
    data = {}
    for n in range(700, 2500):
        data[n] = {}
        for i in range(30,100):
            data[n][i] = 0

    for n in range(700, 2500):
        myPiece = bachBundle[n].parse()

        for m in myPiece.flat.getElementsByClass('Note'):
                data[n][m.midi] +=1

        print 'Number %i' % n

    new_data = np.array([data[700].values()])
    new_data /= np.sum(new_data)

    for index in range(700, 2500):
        temp = np.array([data[index].values()]).astype(np.float64)
        temp /= np.sum(temp)

        new_data =  np.concatenate( (new_data,  temp )  , axis=0)

    print 'Statistics gathered!'

    X = new_data
    d = np.concatenate((save,X))
    n_components=2
    pca = PCA(n_components=n_components).fit(d)
    pca.components_.tofile(out_fp)

    print '{} PCs written to {}'.format(n_components, out_fp)

 def testSearch11(self):
     searchResults = corpus.search('mode phry(.*)', field='keySignature')
     self.assertEqual(len(searchResults) >= 9, True)

 def testSearch06(self):
     searchResults = corpus.search('haydn', field='composer')
     self.assertEqual(len(searchResults), 0)
     searchResults = corpus.search('haydn|bach', field='composer')
     self.assertEqual(len(searchResults) >= 16, True)

 def testSearch02(self):
     searchResults = corpus.search("Sichuan", field="locale")
     self.assertEqual(len(searchResults), 47)

    def testSearch10(self):
        from music21 import key

        ks = key.KeySignature(3)
        searchResults = corpus.search(str(ks), field="keySignature")
        self.assertEqual(len(searchResults) >= 32, True, len(searchResults))

 def testSearch01(self):
     searchResults = corpus.search("china", field="locale")
     self.assertEqual(len(searchResults) > 1200, True)

 def testSearch07(self):
     searchResults = corpus.search("canon")
     self.assertEqual(len(searchResults) >= 1, True)

 def testSearch09(self):
     searchResults = corpus.search("3/.", field="timeSignature")
     self.assertEqual(len(searchResults) >= 2200, True)

    def testSearch(self):

        from music21 import corpus, key

        post = corpus.search('china', 'locale')
        self.assertEqual(len(post) > 1200, True)
        
        post = corpus.search('Sichuan', 'locale')
        self.assertEqual(len(post), 47)
        
        post = corpus.search('Taiwan', 'locale')
        self.assertEqual(len(post), 27)
        self.assertEqual(post[0][0][-8:], 'han2.abc') # file
        self.assertEqual(post[0][1], '209') # work number
        
        post = corpus.search('Sichuan|Taiwan', 'locale')
        self.assertEqual(len(post), 74)


        post = corpus.search('bach')
        self.assertEqual(len(post) > 120, True)

        post = corpus.search('haydn', 'composer')
        self.assertEqual(len(post), 0)
        post = corpus.search('haydn|beethoven', 'composer')
        self.assertEqual(len(post) >= 16, True)


        post = corpus.search('canon')
        self.assertEqual(len(post) >= 1, True)

        post = corpus.search('3/8', 'timeSignature')
        self.assertEqual(len(post) > 360, True)

        post = corpus.search('3/.', 'timeSignature')
        self.assertEqual(len(post) >= 2200 , True)


        ks = key.KeySignature(3, 'major')
        post = corpus.search(str(ks), 'keySignature')
        self.assertEqual(len(post) >= 32, True)

        post = corpus.search('sharps (.*), mode phry(.*)', 'keySignature')
        self.assertEqual(len(post) >= 9, True)

        # searching virtual entries
        post = corpus.search('coltrane', 'composer')
        self.assertEqual(len(post) > 0, True)
        # returns items in pairs: url and work number
        self.assertEqual(post[0][0], 'http://static.wikifonia.org/1164/musicxml.mxl')

 def testSearch09(self):
     searchResults = corpus.search('3/.', field='timeSignature')
     self.assertEqual(len(searchResults) >= 2200 , True)