Navigation

← Previous Changeset
Next Changeset →

Changeset 189

Timestamp:

08/16/08 23:14:08 (3 months ago)

Author:

rgrp

Message:

[shakespeare/cli]: add in new stats command and extend/refactor existing search one. At same time remove stats stuff left in search.py and improve test to stand on its own.

Files:

trunk/shakespeare/cli.py (modified) (1 diff)
trunk/shakespeare/search.py (modified) (1 diff)
trunk/shakespeare/tests/functional/test_search.py (modified) (1 diff)
trunk/shakespeare/tests/test_search.py (modified) (1 diff)

Legend:

: Unmodified
: Added
: Removed
: Modified
: Copied
: Moved

trunk/shakespeare/cli.py

 #!/usr/bin/env python
 import cmd
 import os
 import StringIO
 class ShakespeareAdmin(cmd.Cmd):
     """
     TODO: self.verbose option and associated self._print
     """
     def __init__(self, verbose=False):
         # cmd.Cmd is not a new style class
         cmd.Cmd.__init__(self)
         self.verbose = verbose
     prompt = 'The Bard > '
     def run_interactive(self, line=None):
         """Run an interactive session.
         """
         print 'Welcome to shakespeare-admin interactive mode\n'
         self.do_about()
         print 'Type:  "?" or "help" for help on commands.\n'
         while 1:
             try:
                 self.cmdloop()
                 break
             except KeyboardInterrupt:
                 raise
     def do_help(self, line=None):
         cmd.Cmd.do_help(self, line)
     def do_about(self, line=None):
         import shakespeare
         version = shakespeare.__version__
         about = \
 '''Open Shakespeare version %s. Copyright the Open Knowledge Foundation.
 Open Shakespeare is open-knowledge and open-source. See COPYING for details.
 For more information about the package run `info`.
 ''' % version
         print about
     def do_quit(self, line=None):
         sys.exit()
     def do_EOF(self, *args):
         print ''
         sys.exit()
     # =================
     # Commands
     def do_db(self, line=None):
         actions = [ 'create', 'clean', 'rebuild', 'init' ]
         if line is None or line not in actions:
             self.help_db()
             return 1
         import shakespeare.model
         if line == 'init':
             import pkg_resources
             pkg = 'shksprdata'
             meta = pkg_resources.resource_stream(pkg, 'texts/metadata.txt')
             shakespeare.model.Material.load_from_metadata(meta)
         else:
-            shakespeare.model.__dict__[line+'db']()
+            print 'To create db use paster: paster setup-app {config-file}'
     def help_db(self, line=None):
         usage = \
-'''db { create | clean | rebuild | init }
+'''db { create | init }
 '''
         print usage
     def do_gutenberg(self, line=None):
         import shakespeare.gutenberg
         helper = shakespeare.gutenberg.Helper(verbose=True)
         if not line:
             helper.execute()
         elif line == 'print_index':
             import pprint
             pprint.pprint(helper.get_index())
         else:
             msg = 'Unknown argument %s' % line
             raise Exception(msg)
     def help_gutenberg(self, line=None):
         usage = \
 """
 Download and process all Project Gutenberg shakespeare texts"""
         print usage
     def do_moby(self, line=None):
         import shakespeare.moby
         helper = shakespeare.moby.Helper(verbose=True)
         if not line:
             helper.execute()
         elif line == 'print_index':
             import pprint
             pprint.pprint(helper.get_index())
         else:
             msg = 'Unknown argument %s' % line
             raise Exception(msg)
     def help_moby(self, line=None):
         usage = \
 '''
 Download and process all Moby/Bosak shakespeare texts'''
         print usage
     def _init_index(self):
         import shakespeare.index
         self._index = shakespeare.index.all
     def _filter_index(self, line):
         """Filter items in index return only those whose id (url) is in line
         If line is empty or None return all items
         """
         if line:
             textsToAdd = []
             textNames = line.split()
             for item in self._index:
                 if item.name in textNames:
                     textsToAdd.append(item)
             return textsToAdd
         else:
             self._init_index()
             return self._index
     def do_index(self, line):
         self._init_index()
         header = \
 '''          +-------------------+
           | Index of Material |
           +-------------------+
 '''
         print header
         for row in self._index:
             print row.name.ljust(35), row.title
     def help_index(self, line=None):
         usage = \
 '''Print index of Shakespeare texts to stdout'''
         print usage
-    def do_concordance(self, line=None):
-        self._init_index()
-        print 'Making concordance (this may take some time ...):'
-        from shakespeare.concordance import ConcordanceBuilder
-        import time
-        start = end = 0
-        start = time.time()
-        cc = ConcordanceBuilder()
-        textsToAdd = []
-        if line is not None:
-            textsToAdd = self._filter_index(line)
-        else:
-            def gut_non_folio(material):
-                return '_gut' in material.name and 'gut_f' not in material.name
-            textsToAdd = filter(gut_non_folio, self._index)
-        for item in textsToAdd:
-            print 'Adding: %s (%s)' % (item.name, item.title)
-            cc.add_text(item.name)
-        end = time.time()
-        timetaken = end - start
-        print 'Finished. Time taken was %ss' % timetaken
-    def help_concordance(self, line=None):
-        usage = \
-'''Create a concordance
-If no arguments supplied then use all non-folio gutenberg shakespeare texts.
-Otherwise arguments should be a space seperated list of work name ids
-'''
-        print usage
     def do_runserver(self, line=None):
         self.help_runserver()
     def help_runserver(self, line=None):
         usage = \
 '''This command has been DEPRECATED.
 Please use `paster serve` to run a server now, e.g.::
     paster serve <my-config.ini>
 '''
         print usage
     def do_info(self, line=None):
         import shakespeare
         info = shakespeare.__doc__
         print
         print '       ## Open Shakespeare ##'
         print info
     def help_info(self, line=None):
         print 'Information about this package.'
-    def do_search_add(self, line=None):
+    def _parse_line(self, line):
-        path = line.strip()
+        line = line.strip()
-        if not os.path.exists(path):
+        args = line.split()
-            print '"%s" is not an existent path' % path
+        action = ''
-            return 1
+        remainder = ''
-        if os.path.isdir(path):
+        if len(args) > 0:
-            fns = os.listdir(path)
+            action = args[0]
-            fns = filter(lambda x: x.endswith('.txt'), fns)
+        if len(args) > 1:
-            works = [ os.path.join(path, fn) for fn in fns ]
+            remainder = ' '.join(args[1:])
-        else:
+        return (action, remainder)
+            works = [ path ]
+    def do_search(self, line):
         import shakespeare.search
         index = shakespeare.search.SearchIndex.default_index()
+        for work in works:
-            if self.verbose:
+        action, extra = self._parse_line(line)
-                print 'Processing %s' % work
+        if action == 'addpath':
-            fileobj = open(work)
+            index.add_from_path(extra)
+        elif action == 'query':
+            results = index.search(extra)
+            print index.print_matches(results)
+        elif action == 'addtext':
+            import shakespeare.model as model
+            text = model.Material.byName(extra)
+            fileobj = text.get_text()
             index.add_item(fileobj)
+        elif action == 'init':
-    def help_search_add(self, line=None):
+            self._init_index()
-        info = '''search_add {path}
+            for text in self._index:
+                fileobj = text.get_text()
-Add contents of {path} (file itself or all text files in directory if
+                index.add_item(fileobj)
-directory) to the search index.'''
+        else:
+            print 'Unrecognized action: %s' % action
+            self.help_search()
+            return 1
+    def help_search(self, line=None):
+        info = \
+'''
+search addpath {path}
+    - Add contents of {path} (file itself or all text files in directory if
+      directory) to the search index.
+search addtext {name}
+    - Add db text named {name} to search index.
+search query {query}
+    - Query search index with {query}.
+search init
+    - Add all texts in DB to index.
+'''
         print info
-    def do_search_add_all(self):
+    def do_stats(self, line):
-        # TODO: automatically add all texts listed in index
+        action, extra = self._parse_line(line)
+        pass
+        import shakespeare.stats
-    def do_search(self, line=None):
+        stats = shakespeare.stats.Stats()
-        import shakespeare.search
+        if action == 'init':
-        index = shakespeare.search.SearchIndex.default_index()
+            self._init_index()
-        query = line.strip()
+            for text in self._index:
-        if not query:
+                stats.statsify(text, text.get_text())
-            print 'No search term supplied.'
+        elif action == 'addtext':
+            import shakespeare.model as model
+            text = model.Material.byName(extra)
+            stats.statsify(text, text.get_text())
+        elif action == 'show':
+            textstats = stats.text_stats(extra)
+            for s in textstats:
+                print s.word, s.freq
+        else:
+            print 'Unrecognized action: %s' % action
+            self.help_stats()
             return 1
+        matches = index.search(query)
-        print "%i results found." % matches.get_matches_estimated()
+    def help_stats(self, line=None):
-        print "Results 1-%i:" % matches.size()
+        info = \
+'''
-        for m in matches:
+stats addtext {name}
-            print
+    - Add db text named {name} to stats index.
+            print '%i: %i%% docid=%i' % (m.rank + 1, m.percent, m.docid)
-            print m.document.get_data()
+stats show {name}
+    - Query stats index with {query}.
+    def help_search(self, line=None):
-        info = 'Supply a query with which to search the search index.'
+stats init
+    - Prepare statistics for all texts in DB.
+'''
         print info
 def main():
     import optparse
     usage = \
 '''%prog [options] <command>
 Run about or help for details.'''
     parser = optparse.OptionParser(usage)
     parser.add_option('-v', '--verbose', dest='verbose', help='Be verbose',
             action='store_true', default=False)
     options, args = parser.parse_args()
     if len(args) == 0:
         parser.print_help()
         return 1
     else:
         cmd = ShakespeareAdmin(verbose=options.verbose)
         args = ' '.join(args)
         args = args.replace('-','_')
         cmd.onecmd(args)

trunk/shakespeare/search.py

 '''Support for indexing and searching texts using xapian.
 Architecture
 ============
 For information on theoretical structure of Xapain see:
 http://xapian.org/docs/intro_ir.html
 For basic demo python code see: http://xapian.org/docs/bindings/python/
 For helpful example of using Xapian in python (including metadata, add_post
 etc) see:
   * http://www.thesamet.com/blog/2007/02/04/pumping-up-your-applications-with-xapian-full-text-search/
   * http://www.rkblog.rk.edu.pl/w/p/xapian-python/
 Here we discuss how we can use Xapian in OS. Two main tasks:
 . Do search
 . Produce statistics
 Second task just requires stemming support, first requires full Xapian
 facilities. Main question for indexing is:
   * What is our atomization level. I.e. what are 'documents' we index? Is it:
     * A whole poem or play
     * Is it a paragraph within a work
     * Is it a character's whole speech?
 TODO:
     * add metadata (e.g. which character is speaking, work id ...)
 '''
 import os
 import re
 import xapian
 class SearchIndex(object):
     def __init__(self, index_dir):
         self.index_dir = index_dir
     @classmethod
     def config_index_dir(self):
         '''Get the search index directory specified in the config.'''
         import shakespeare
         conf = shakespeare.conf()
         index_dir = conf['search_index_dir']
         return index_dir
     @classmethod
     def default_index(self):
         '''Return a SearchIndex instance initialized with the path specified in
         the configuration file.
         '''
         index_dir = self.config_index_dir()
         if not os.path.exists(index_dir):
             os.makedirs(index_dir)
         return SearchIndex(index_dir)
-    @classmethod
-    def get_stats(self, fileobj):
-        '''Get statistics on text in fileobj.
-        Words are stemmed so that e.g. love and loved count as the same word.
-        '''
-        # (?) maybe could use xapian.TermGenerator to split document
-        WORD_RE = re.compile('\\w{1,32}', re.U)
-        stemmer = xapian.Stem('english')
-        results = {}
-        text = fileobj.read()
-        text = text.encode('utf8')
-        for term in WORD_RE.finditer(text):
-            word = term.group()
-            word = word.lower()
-            stemmed_word = stemmer(word)
-            results[stemmed_word] = results.get(stemmed_word, 0) + 1
-        return results
     def add_item(self, fileobj):
         document = xapian.WritableDatabase (self.index_dir, xapian.DB_CREATE_OR_OPEN)
         indexer = xapian.TermGenerator()
         stemmer = xapian.Stem("english")
         indexer.set_stemmer(stemmer)
         para = ''
         try:
             for line in fileobj:
                 line = line.strip()
                 if line == '':
                     if para != '':
                         doc = xapian.Document()
                         doc.set_data(para)
                         indexer.set_document(doc)
                         # this *will* include positional information
                         indexer.index_text(para)
                         # Add the document to the database.
                         document.add_document(doc)
                         para = ''
                 else:
                     if para != '':
                         para += '\n'
                     para += line
         except StopIteration:
             # TODO: what is happening here?
             pass
     def search(self, query_string):
         # Open the database for searching.
         database = xapian.Database(self.index_dir)
             # Start an enquire session.
         enquire = xapian.Enquire(database)
         # Parse the query string to produce a Xapian::Query object.
         qp = xapian.QueryParser()
         stemmer = xapian.Stem("english")
         qp.set_stemmer(stemmer)
         qp.set_database(database)
         qp.set_stemming_strategy(xapian.QueryParser.STEM_SOME)
         query = qp.parse_query(query_string)
         print "Parsed query is: %s" % query.get_description()
          # Find the top 10 results for the query.
         enquire.set_query(query)
         # get search results offset, offset+count
         offset = 0
         count = 10
         matches = enquire.get_mset(offset, count)
         return matches
+    def add_from_path(self, path):
+        '''Add contents of {path} (file itself or all text files in directory
+        if directory) to the search index.'''
+        path = path.strip()
+        if not os.path.exists(path):
+            print '"%s" is not an existent path' % path
+            return 1
+        if os.path.isdir(path):
+            fns = os.listdir(path)
+            fns = filter(lambda x: x.endswith('.txt'), fns)
+            works = [ os.path.join(path, fn) for fn in fns ]
+        else:
+            works = [ path ]
+        for work in works:
+            if self.verbose:
+                print 'Processing %s' % work
+            fileobj = open(work)
+            self.index.add_item(fileobj)
     @classmethod
     def print_matches(self, matches):
         # Display the results.
-        print "%i results found." % matches.get_matches_estimated()
+        msg = '%i results found.' % matches.get_matches_estimated()
-        print "Results 1-%i:" % matches.size()
+        msg += 'Results 1-%i:' % matches.size()
         for m in matches:
-            print "%i: %i%% docid=%i [%s]" % (m.rank + 1, m.percent, m.docid, m.document.get_data())
+            msg += '\n'
+            msg += '%i: %i%% docid=%i' % (m.rank + 1, m.percent, m.docid)
+            msg += '\n'
+            msg += m.document.get_data()
+            msg += '\n'
+        return msg

trunk/shakespeare/tests/functional/test_search.py

+import StringIO
 from shakespeare.tests import *
+import shakespeare.search
 class TestSearchController(TestController):
+    def setUp(self):
+        # TODO: remove this item from index in tearDown
+        text = make_fixture()
+        sindex = shakespeare.search.SearchIndex.default_index()
+        sindex.add_item(StringIO.StringIO(text.content))
     def test_index(self):
         url = url_for(controller='search')
         res = self.app.get(url)
         assert "Search" in res
     def test_search(self):
         url = url_for(controller='search')
         res = self.app.get(url)
         form = res.forms[0]
-        # for this to work need to have added phoenix to the index
+        form['query'] = 'summer'
+        # TODO: put this in setUp or something ...
+        form['query'] = 'Phoenix'
         res = form.submit()
         assert 'Search Results' in res
-        assert 'Phoenix' in res
+        assert 'Shall I compare thee' in res

trunk/shakespeare/tests/test_search.py

 import os
 import shutil
 import tempfile
 import StringIO
 import shakespeare.search
 class TestSearch:
     # break up a little to make indexing more interesting
     text = \
 '''
 Shall I compare thee to a summer's day?
 Thou art more lovely and more temperate:
 Rough winds do shake the darling buds of May,
 And summer's lease hath all too short a date:
 Sometime too hot the eye of heaven shines,
 And often is his gold complexion dimm'd,
 And every fair from fair sometime declines,
 By chance, or nature's changing course untrimm'd:
 But thy eternal summer shall not fade,
 Nor lose possession of that fair thou ow'st,
 Nor shall death brag thou wander'st in his shade,
 When in eternal lines to time thou grow'st,
   So long as men can breathe, or eyes can see,
   So long lives this, and this gives life to thee.
 '''
     def setUp(self):
         basetmp = tempfile.gettempdir()
         self.tmpdir = os.path.join(basetmp, 'openshkspr-search')
         # we leave directory in existence to help with debugging
         if os.path.exists(self.tmpdir):
             shutil.rmtree(self.tmpdir)
         os.makedirs(self.tmpdir)
         self.index = shakespeare.search.SearchIndex(self.tmpdir)
     def test_add_item(self):
         self.index.add_item(StringIO.StringIO(self.text))
     def test_search(self):
         self.index.add_item(StringIO.StringIO(self.text))
         out = self.index.search('summer')
         assert len(out) == 2
         mset1 = out[1]
         # 'But thy eternal summer ...
         exp = "But thy eternal summer shall not fade,\nNor lose possession of that fair thou ow'st,"
         assert mset1.document.get_data().startswith(exp)
         out = self.index.search('rough')
         assert len(out) == 1
-    def test_get_stats(self):
-        simpletext = 'Death death dead love loved loving'
-        out = self.index.get_stats(StringIO.StringIO(simpletext))
-        assert len(out) == 3
-        assert out['love'] == 3
-        assert out['death'] == 2
-        assert out['dead'] == 1

Download in other formats: