Commit 454d0933 authored by Sam Thursfield's avatar Sam Thursfield

functional-tests: Split 310-fts test into 3 parts

This test takes a long time, so to provide better feedback and to avoid
timeouts let's split it into 3.
parent 482ba629
#!/usr/bin/python
#-*- coding: utf-8 -*-
# Copyright (C) 2010, Nokia (ivan.frade@nokia.com)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
#
# TODO:
# These tests are for files... we need to write them for folders!
#
"""
Monitor a directory, copy/move/remove/update text files and check that
the text contents are updated accordingly in the indexes.
"""
import os
import shutil
import locale
import time
import unittest2 as ut
from common.utils.helpers import log
from common.utils.minertest import CommonTrackerMinerFTSTest, MINER_TMP_DIR, uri, path, DEFAULT_TEXT
from common.utils import configuration as cfg
class MinerFTSBasicTest (CommonTrackerMinerFTSTest):
"""
Tests different contents in a single file
"""
def test_01_single_word (self):
TEXT = "automobile"
self.basic_test (TEXT, TEXT)
def test_02_multiple_words (self):
TEXT = "automobile with unlimited power"
self.set_text (TEXT)
results = self.search_word ("automobile")
self.assertEquals (len (results), 1)
self.assertIn (uri (self.testfile), results)
results = self.search_word ("unlimited")
self.assertEquals (len (results), 1)
self.assertIn (uri (self.testfile), results)
def test_03_long_word (self):
# TEXT is longer than the 20 characters specified in the fts configuration
TEXT = "fsfsfsdfskfweeqrewqkmnbbvkdasdjefjewriqjfnc"
self.set_text (TEXT)
results = self.search_word (TEXT)
self.assertEquals (len (results), 0)
def test_04_non_existent_word (self):
TEXT = "This a trick"
self.set_text (TEXT)
results = self.search_word ("trikc")
self.assertEquals (len (results), 0)
def test_05_word_in_multiple_files (self):
# Safeguard, in the case we modify the DEFAULT_TEXT later...
assert "content" in DEFAULT_TEXT
self.set_text (DEFAULT_TEXT)
results = self.search_word ("content")
self.assertEquals (len (results), 4)
self.assertIn ( uri (self.testfile), results)
self.assertIn ( uri ("test-monitored/file1.txt"), results)
self.assertIn ( uri ("test-monitored/dir1/file2.txt"), results)
self.assertIn ( uri ("test-monitored/dir1/dir2/file3.txt"), results)
def test_06_word_multiple_times_in_file (self):
TEXT = "automobile is red. automobile is big. automobile is great!"
self.basic_test (TEXT, "automobile")
def test_07_sentence (self):
TEXT = "plastic is fantastic"
self.basic_test (TEXT, TEXT)
def test_08_partial_sentence (self):
TEXT = "plastic is fantastic"
self.basic_test (TEXT, "is fantastic")
def test_09_strange_word (self):
# FIXME Not sure what are we testing here
TEXT = "'summer.time'"
self.basic_test (TEXT, "summer.time")
# Skip the test 'search for .'
def test_10_mixed_letters_and_numbers (self):
TEXT = "abc123"
self.basic_test (TEXT, "abc123")
def test_11_ignore_numbers (self):
TEXT = "palabra 123123"
self.set_text (TEXT)
results = self.search_word ("123123")
self.assertEquals (len (results), 0)
if __name__ == "__main__":
ut.main ()
......@@ -33,156 +33,11 @@ import time
import unittest2 as ut
from common.utils.helpers import log
from common.utils.minertest import CommonTrackerMinerTest, MINER_TMP_DIR, uri, path, DEFAULT_TEXT
from common.utils.minertest import CommonTrackerMinerFTSTest, MINER_TMP_DIR, uri, path, DEFAULT_TEXT
from common.utils import configuration as cfg
class CommonMinerFTS (CommonTrackerMinerTest):
"""
Superclass to share methods. Shouldn't be run by itself.
"""
def prepare_directories (self):
# Override content from the base class
pass
def setUp (self):
self.testfile = "test-monitored/miner-fts-test.txt"
if os.path.exists (path (self.testfile)):
os.remove (path (self.testfile))
super(CommonMinerFTS, self).setUp()
def set_text (self, text):
exists = os.path.exists(path(self.testfile))
f = open (path (self.testfile), "w")
f.write (text)
f.close ()
if exists:
subject_id = self.tracker.get_resource_id(uri(self.testfile))
self.tracker.await_property_changed(
subject_id=subject_id, property_uri='nie:plainTextContent')
else:
self.tracker.await_resource_inserted(
rdf_class='nfo:Document', url=uri(self.testfile),
required_property='nie:plainTextContent')
self.tracker.reset_graph_updates_tracking()
def search_word (self, word):
"""
Return list of URIs with the word in them
"""
log ("Search for: %s" % word)
results = self.tracker.query ("""
SELECT ?url WHERE {
?u a nfo:TextDocument ;
nie:url ?url ;
fts:match '%s'.
}
""" % (word))
return [r[0] for r in results]
def basic_test (self, text, word):
"""
Save the text on the testfile, search the word
and assert the testfile is only result.
Be careful with the default contents of the text files
( see common/utils/minertest.py DEFAULT_TEXT )
"""
self.set_text (text)
results = self.search_word (word)
self.assertEquals (len (results), 1)
self.assertIn ( uri (self.testfile), results)
def _query_id (self, uri):
query = "SELECT tracker:id(?urn) WHERE { ?urn nie:url \"%s\". }" % uri
result = self.tracker.query (query)
assert len (result) == 1
return int (result[0][0])
class MinerFTSBasicTest (CommonMinerFTS):
"""
Tests different contents in a single file
"""
def test_01_single_word (self):
TEXT = "automobile"
self.basic_test (TEXT, TEXT)
def test_02_multiple_words (self):
TEXT = "automobile with unlimited power"
self.set_text (TEXT)
results = self.search_word ("automobile")
self.assertEquals (len (results), 1)
self.assertIn (uri (self.testfile), results)
results = self.search_word ("unlimited")
self.assertEquals (len (results), 1)
self.assertIn (uri (self.testfile), results)
def test_03_long_word (self):
# TEXT is longer than the 20 characters specified in the fts configuration
TEXT = "fsfsfsdfskfweeqrewqkmnbbvkdasdjefjewriqjfnc"
self.set_text (TEXT)
results = self.search_word (TEXT)
self.assertEquals (len (results), 0)
def test_04_non_existent_word (self):
TEXT = "This a trick"
self.set_text (TEXT)
results = self.search_word ("trikc")
self.assertEquals (len (results), 0)
def test_05_word_in_multiple_files (self):
# Safeguard, in the case we modify the DEFAULT_TEXT later...
assert "content" in DEFAULT_TEXT
self.set_text (DEFAULT_TEXT)
results = self.search_word ("content")
self.assertEquals (len (results), 4)
self.assertIn ( uri (self.testfile), results)
self.assertIn ( uri ("test-monitored/file1.txt"), results)
self.assertIn ( uri ("test-monitored/dir1/file2.txt"), results)
self.assertIn ( uri ("test-monitored/dir1/dir2/file3.txt"), results)
def test_06_word_multiple_times_in_file (self):
TEXT = "automobile is red. automobile is big. automobile is great!"
self.basic_test (TEXT, "automobile")
def test_07_sentence (self):
TEXT = "plastic is fantastic"
self.basic_test (TEXT, TEXT)
def test_08_partial_sentence (self):
TEXT = "plastic is fantastic"
self.basic_test (TEXT, "is fantastic")
def test_09_strange_word (self):
# FIXME Not sure what are we testing here
TEXT = "'summer.time'"
self.basic_test (TEXT, "summer.time")
# Skip the test 'search for .'
def test_10_mixed_letters_and_numbers (self):
TEXT = "abc123"
self.basic_test (TEXT, "abc123")
def test_11_ignore_numbers (self):
TEXT = "palabra 123123"
self.set_text (TEXT)
results = self.search_word ("123123")
self.assertEquals (len (results), 0)
class MinerFTSFileOperationsTest (CommonMinerFTS):
class MinerFTSFileOperationsTest (CommonTrackerMinerFTSTest):
"""
Move, update, delete the files and check the text indexes are updated accordingly.
"""
......@@ -285,50 +140,6 @@ class MinerFTSFileOperationsTest (CommonMinerFTS):
# skip test for a file in a hidden directory
class MinerFTSStopwordsTest (CommonMinerFTS):
"""
Search for stopwords in a file
"""
def __get_some_stopwords (self):
langcode, encoding = locale.getdefaultlocale ()
if "_" in langcode:
langcode = langcode.split ("_")[0]
stopwordsfile = os.path.join (cfg.DATADIR, "tracker", "stop-words", "stopwords." + langcode)
if not os.path.exists (stopwordsfile):
self.skipTest ("No stopwords for the current locale ('%s' doesn't exist)" % (stopwordsfile))
return []
stopwords = []
counter = 0
for line in open (stopwordsfile, "r"):
if len (line) > 4:
stopwords.append (line[:-1])
counter += 1
if counter > 5:
break
return stopwords
def test_01_stopwords (self):
stopwords = self.__get_some_stopwords ()
TEXT = " ".join (["this a completely normal text automobile"] + stopwords)
self.set_text (TEXT)
results = self.search_word ("automobile")
self.assertEquals (len (results), 1)
log ("Stopwords: %s" % stopwords)
for i in range (0, len (stopwords)):
results = self.search_word (stopwords[i])
self.assertEquals (len (results), 0)
## FIXME add all the special character tests!
## http://git.gnome.org/browse/tracker/commit/?id=81c0d3bd754a6b20ac72323481767dc5b4a6217b
if __name__ == "__main__":
ut.main ()
#!/usr/bin/python
#-*- coding: utf-8 -*-
# Copyright (C) 2010, Nokia (ivan.frade@nokia.com)
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this library; if not, write to the
# Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor,
# Boston, MA 02110-1301, USA.
#
# TODO:
# These tests are for files... we need to write them for folders!
#
"""
Monitor a directory, copy/move/remove/update text files and check that
the text contents are updated accordingly in the indexes.
"""
import os
import shutil
import locale
import time
import unittest2 as ut
from common.utils.helpers import log
from common.utils.minertest import CommonTrackerMinerFTSTest, MINER_TMP_DIR, uri, path, DEFAULT_TEXT
from common.utils import configuration as cfg
class MinerFTSStopwordsTest (CommonTrackerMinerFTSTest):
"""
Search for stopwords in a file
"""
def __get_some_stopwords (self):
langcode, encoding = locale.getdefaultlocale ()
if "_" in langcode:
langcode = langcode.split ("_")[0]
stopwordsfile = os.path.join (cfg.DATADIR, "tracker", "stop-words", "stopwords." + langcode)
if not os.path.exists (stopwordsfile):
self.skipTest ("No stopwords for the current locale ('%s' doesn't exist)" % (stopwordsfile))
return []
stopwords = []
counter = 0
for line in open (stopwordsfile, "r"):
if len (line) > 4:
stopwords.append (line[:-1])
counter += 1
if counter > 5:
break
return stopwords
def test_01_stopwords (self):
stopwords = self.__get_some_stopwords ()
TEXT = " ".join (["this a completely normal text automobile"] + stopwords)
self.set_text (TEXT)
results = self.search_word ("automobile")
self.assertEquals (len (results), 1)
log ("Stopwords: %s" % stopwords)
for i in range (0, len (stopwords)):
results = self.search_word (stopwords[i])
self.assertEquals (len (results), 0)
## FIXME add all the special character tests!
## http://git.gnome.org/browse/tracker/commit/?id=81c0d3bd754a6b20ac72323481767dc5b4a6217b
if __name__ == "__main__":
ut.main ()
......@@ -17,7 +17,10 @@ config_SCRIPTS = \
tests = \
300-miner-basic-ops.py \
301-miner-resource-removal.py
301-miner-resource-removal.py \
310-fts-basic.py \
311-fts-file-operations.py \
312-fts-stopwords.py \
400-extractor-metadata.py \
410-extractor-decorator.py \
500-writeback.py \
......
......@@ -18,6 +18,7 @@
# 02110-1301, USA.
#
from common.utils import configuration as cfg
from common.utils.helpers import log
from common.utils.system import TrackerSystemAbstraction
import unittest2 as ut
......@@ -120,3 +121,70 @@ class CommonTrackerMinerTest (ut.TestCase):
def assertResourceMissing (self, urn):
if self.tracker.ask ("ASK { <%s> a rdfs:Resource }" % urn) == True:
self.fail ("Resource <%s> should not exist" % urn)
class CommonTrackerMinerFTSTest (CommonTrackerMinerTest):
"""
Superclass to share methods. Shouldn't be run by itself.
"""
def prepare_directories (self):
# Override content from the base class
pass
def setUp (self):
self.testfile = "test-monitored/miner-fts-test.txt"
if os.path.exists (path (self.testfile)):
os.remove (path (self.testfile))
super(CommonTrackerMinerFTSTest, self).setUp()
def set_text (self, text):
exists = os.path.exists(path(self.testfile))
f = open (path (self.testfile), "w")
f.write (text)
f.close ()
if exists:
subject_id = self.tracker.get_resource_id(uri(self.testfile))
self.tracker.await_property_changed(
subject_id=subject_id, property_uri='nie:plainTextContent')
else:
self.tracker.await_resource_inserted(
rdf_class='nfo:Document', url=uri(self.testfile),
required_property='nie:plainTextContent')
self.tracker.reset_graph_updates_tracking()
def search_word (self, word):
"""
Return list of URIs with the word in them
"""
log ("Search for: %s" % word)
results = self.tracker.query ("""
SELECT ?url WHERE {
?u a nfo:TextDocument ;
nie:url ?url ;
fts:match '%s'.
}
""" % (word))
return [r[0] for r in results]
def basic_test (self, text, word):
"""
Save the text on the testfile, search the word
and assert the testfile is only result.
Be careful with the default contents of the text files
( see common/utils/minertest.py DEFAULT_TEXT )
"""
self.set_text (text)
results = self.search_word (word)
self.assertEquals (len (results), 1)
self.assertIn ( uri (self.testfile), results)
def _query_id (self, uri):
query = "SELECT tracker:id(?urn) WHERE { ?urn nie:url \"%s\". }" % uri
result = self.tracker.query (query)
assert len (result) == 1
return int (result[0][0])
......@@ -9,7 +9,9 @@ config_json = configure_file(
functional_tests = [
'300-miner-basic-ops',
'301-miner-resource-removal',
'310-fts-indexing',
'310-fts-basic',
'311-fts-file-operations',
'312-fts-stopwords',
'410-extractor-decorator',
'500-writeback',
'501-writeback-details',
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment