Commit df0fa863 authored by Sam Thursfield's avatar Sam Thursfield

functional-tests: Fix more issues in awaiting for property changes

We actually get a GraphUpdated signal per RDF class, and we need to
filter out those we don't care about.

This was breaking tests in some cases as they would get double
notifications and get ahead of themselves. For example the
410-extractor-decorator test would wait for changes to nie:title on
an nmm:MusicPiece resource. But music files also have class nfo:Audio
and so there would be two GraphUpdated signals for each change, one
for the nmm:MusicPiece class and one for the nfo:Audio class.
parent 33fdde3d
......@@ -33,6 +33,8 @@ import unittest2 as ut
from common.utils.helpers import log
from common.utils.minertest import CommonTrackerMinerTest
NFO_TEXT_DOCUMENT = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#TextDocument'
class MinerCrawlTest (CommonTrackerMinerTest):
"""
Test cases to check if miner is able to monitor files that are created, deleted or moved
......@@ -98,7 +100,7 @@ class MinerCrawlTest (CommonTrackerMinerTest):
dest = os.path.join (self.datadir, "test-monitored", "file0.txt")
shutil.copyfile (source, dest)
dest_id, dest_urn = self.system.store.await_resource_inserted ('nfo:TextDocument', self.uri(dest))
dest_id, dest_urn = self.system.store.await_resource_inserted (NFO_TEXT_DOCUMENT, self.uri(dest))
# verify if miner indexed this file.
result = self.__get_text_documents ()
......@@ -145,7 +147,7 @@ class MinerCrawlTest (CommonTrackerMinerTest):
dest = os.path.join (self.datadir, "test-monitored", "dir1", "dir2", "file-test04.txt")
shutil.copyfile (source, dest)
dest_id, dest_urn = self.system.store.await_resource_inserted ('nfo:TextDocument', self.uri(dest))
dest_id, dest_urn = self.system.store.await_resource_inserted (NFO_TEXT_DOCUMENT, self.uri(dest))
result = self.__get_text_documents ()
self.assertEquals (len (result), 4)
......@@ -168,7 +170,7 @@ class MinerCrawlTest (CommonTrackerMinerTest):
source = os.path.join (self.datadir, "test-no-monitored", "file0.txt")
dest = os.path.join (self.datadir, "test-monitored", "dir1", "file-test05.txt")
shutil.move (source, dest)
dest_id, dest_urn = self.system.store.await_resource_inserted ('nfo:TextDocument', self.uri(dest))
dest_id, dest_urn = self.system.store.await_resource_inserted (NFO_TEXT_DOCUMENT, self.uri(dest))
result = self.__get_text_documents ()
self.assertEquals (len (result), 4)
......@@ -205,7 +207,7 @@ class MinerCrawlTest (CommonTrackerMinerTest):
# Restore the file
shutil.move (dest, source)
self.system.store.await_resource_inserted ('nfo:TextDocument', self.uri(source))
self.system.store.await_resource_inserted (NFO_TEXT_DOCUMENT, self.uri(source))
self.assertEquals (3, self.tracker.count_instances ("nfo:TextDocument"))
......@@ -269,7 +271,7 @@ class MinerCrawlTest (CommonTrackerMinerTest):
f = open (victim, "w")
f.write ("Don't panic, everything is fine")
f.close ()
self.system.store.await_resource_inserted ('nfo:TextDocument', self.uri(victim))
self.system.store.await_resource_inserted (NFO_TEXT_DOCUMENT, self.uri(victim))
def test_09_deletion_directory (self):
"""
......@@ -297,7 +299,7 @@ class MinerCrawlTest (CommonTrackerMinerTest):
writer = open (filename, "w")
writer.write ("Don't panic, everything is fine")
writer.close ()
self.system.store.await_resource_inserted ('nfo:TextDocument', self.uri(f))
self.system.store.await_resource_inserted (NFO_TEXT_DOCUMENT, self.uri(f))
# Check everything is fine
result = self.__get_text_documents ()
......
......@@ -31,6 +31,10 @@ import os
import unittest2 as ut
NFO_DOCUMENT = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#Document'
NMM_MUSIC_PIECE = 'http://www.tracker-project.org/temp/nmm#MusicPiece'
class MinerResourceRemovalTest (CommonTrackerMinerTest):
def prepare_directories (self):
......@@ -46,7 +50,7 @@ class MinerResourceRemovalTest (CommonTrackerMinerTest):
self.tracker.update (sparql)
return self.tracker.await_resource_inserted (rdf_class = 'nmm:MusicPiece',
return self.tracker.await_resource_inserted (rdf_class = NMM_MUSIC_PIECE,
title = title)
def create_test_file (self, file_name):
......@@ -56,7 +60,7 @@ class MinerResourceRemovalTest (CommonTrackerMinerTest):
file.write ("Test")
file.close ()
return self.tracker.await_resource_inserted (rdf_class = 'nfo:Document',
return self.tracker.await_resource_inserted (rdf_class = NFO_DOCUMENT,
url = self.uri(file_name))
def test_01_file_deletion (self):
......@@ -72,8 +76,8 @@ class MinerResourceRemovalTest (CommonTrackerMinerTest):
os.unlink (self.path ("test-monitored/test_1.txt"))
self.tracker.await_resource_deleted (file_1_id)
self.tracker.await_resource_deleted (ie_1_id,
self.tracker.await_resource_deleted (NFO_DOCUMENT, file_1_id)
self.tracker.await_resource_deleted (NFO_DOCUMENT, ie_1_id,
"Associated logical resource failed to be deleted " \
"when its containing file was removed.")
......
......@@ -37,6 +37,9 @@ from common.utils.minertest import CommonTrackerMinerFTSTest, DEFAULT_TEXT
from common.utils import configuration as cfg
NFO_TEXT_DOCUMENT = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#TextDocument'
class MinerFTSFileOperationsTest (CommonTrackerMinerFTSTest):
"""
Move, update, delete the files and check the text indexes are updated accordingly.
......@@ -51,7 +54,7 @@ class MinerFTSFileOperationsTest (CommonTrackerMinerFTSTest):
id = self._query_id (self.uri (self.testfile))
os.remove (self.path (self.testfile))
self.tracker.await_resource_deleted (id)
self.tracker.await_resource_deleted (NFO_TEXT_DOCUMENT, id)
results = self.search_word ("automobile")
self.assertEquals (len (results), 0)
......@@ -125,7 +128,7 @@ class MinerFTSFileOperationsTest (CommonTrackerMinerFTSTest):
self.assertEquals (len (results), 0)
shutil.copyfile (self.path (TEST_16_SOURCE), self.path (TEST_16_DEST))
self.tracker.await_resource_inserted (rdf_class = 'nfo:Document',
self.tracker.await_resource_inserted (rdf_class = NFO_TEXT_DOCUMENT,
url = self.uri(TEST_16_DEST),
required_property = 'nie:plainTextContent')
......
......@@ -42,7 +42,7 @@ CORRUPT_FILE = os.path.join(
VALID_FILE = os.path.join(
os.path.dirname(__file__), 'test-extraction-data', 'audio',
'audio-test-1.mp3')
VALID_FILE_CLASS = 'nmm:MusicPiece'
VALID_FILE_CLASS = 'http://www.tracker-project.org/temp/nmm#MusicPiece'
VALID_FILE_TITLE = 'Simply Juvenile'
TRACKER_EXTRACT_FAILURE_DATA_SOURCE = 'tracker:extractor-failure-data-source'
......@@ -80,7 +80,6 @@ class ExtractorDecoratorTest(ut.TestCase):
shutil.rmtree(self.datadir)
@ut.skip("Currently fails; possible regression")
def test_reextraction(self):
"""Tests whether known files are still re-extracted on user request."""
miner_fs = self.system.miner_fs
......@@ -96,7 +95,7 @@ class ExtractorDecoratorTest(ut.TestCase):
store.update(
'DELETE { <%s> nie:title ?title }'
' WHERE { <%s> nie:title ?title }' % (file_urn, file_urn))
store.await_property_changed(file_id, 'nie:title')
store.await_property_changed(VALID_FILE_CLASS, file_id, 'nie:title')
assert not store.ask('ASK { <%s> nie:title ?title }' % file_urn)
log("Sending re-index request")
......@@ -105,7 +104,7 @@ class ExtractorDecoratorTest(ut.TestCase):
# The extractor should reindex the file and re-add the metadata that we
# deleted, so we should see the nie:title property change.
store.await_property_changed(file_id, 'nie:title')
store.await_property_changed(VALID_FILE_CLASS, file_id, 'nie:title')
title_result = store.query('SELECT ?title { <%s> nie:title ?title }' % file_urn)
assert len(title_result) == 1
......
......@@ -30,6 +30,10 @@ from common.utils.applicationstest import CommonTrackerApplicationTest as Common
from common.utils.helpers import log
NMM_PHOTO = 'http://www.tracker-project.org/temp/nmm#Photo'
NMM_VIDEO = 'http://www.tracker-project.org/temp/nmm#Video'
class TrackerCameraTestSuite (CommonTrackerApplicationTest):
"""
Common functionality for camera tests.
......@@ -146,13 +150,13 @@ class TrackerCameraPicturesApplicationTests (TrackerCameraTestSuite):
# Copy the image to the dest path
self.slowcopy_file (origin_filepath, dest_filepath)
assert os.path.exists (dest_filepath)
dest_id, dest_urn = self.system.store.await_resource_inserted ('nmm:Photo', dest_fileuri)
dest_id, dest_urn = self.system.store.await_resource_inserted (NMM_PHOTO, dest_fileuri)
self.assertEquals (self.get_urn_count_by_url (dest_fileuri), 1)
# Clean the new file so the test directory is as before
log ("Remove and wait")
os.remove (dest_filepath)
self.system.store.await_resource_deleted (dest_id)
self.system.store.await_resource_deleted (NMM_PHOTO, dest_id)
self.assertEquals (self.get_urn_count_by_url (dest_fileuri), 0)
def test_02_camera_picture_geolocation (self):
......@@ -192,13 +196,13 @@ class TrackerCameraPicturesApplicationTests (TrackerCameraTestSuite):
assert os.path.exists (dest_filepath)
# FOURTH, ensure we have only 1 resource
dest_id, dest_urn = self.system.store.await_resource_inserted ('nmm:Photo', dest_fileuri)
dest_id, dest_urn = self.system.store.await_resource_inserted (NMM_PHOTO, dest_fileuri)
self.assertEquals (self.get_urn_count_by_url (dest_fileuri), 1)
# Clean the new file so the test directory is as before
log ("Remove and wait")
os.remove (dest_filepath)
self.system.store.await_resource_deleted (dest_id)
self.system.store.await_resource_deleted (NMM_PHOTO, dest_id)
self.assertEquals (self.get_urn_count_by_url (dest_fileuri), 0)
......@@ -224,13 +228,13 @@ class TrackerCameraVideosApplicationTests (TrackerCameraTestSuite):
# Copy the image to the dest path
self.slowcopy_file (origin_filepath, dest_filepath)
assert os.path.exists (dest_filepath)
dest_id, dest_urn = self.system.store.await_resource_inserted ('nmm:Video', dest_fileuri)
dest_id, dest_urn = self.system.store.await_resource_inserted (NMM_PHOTO, dest_fileuri)
self.assertEquals (self.get_urn_count_by_url (dest_fileuri), 1)
# Clean the new file so the test directory is as before
log ("Remove and wait")
os.remove (dest_filepath)
self.system.store.await_resource_deleted (dest_id)
self.system.store.await_resource_deleted (NMM_PHOTO, dest_id)
self.assertEquals (self.get_urn_count_by_url (dest_fileuri), 0)
......@@ -271,13 +275,13 @@ class TrackerCameraVideosApplicationTests (TrackerCameraTestSuite):
assert os.path.exists (dest_filepath)
# FOURTH, ensure we have only 1 resource
dest_id, dest_urn = self.system.store.await_resource_inserted ('nmm:Video', dest_fileuri)
dest_id, dest_urn = self.system.store.await_resource_inserted (NMM_VIDEO, dest_fileuri)
self.assertEquals (self.get_urn_count_by_url (dest_fileuri), 1)
# Clean the new file so the test directory is as before
log ("Remove and wait")
os.remove (dest_filepath)
self.system.store.await_resource_deleted (dest_id)
self.system.store.await_resource_deleted (NMM_VIDEO, dest_id)
self.assertEquals (self.get_urn_count_by_url (dest_fileuri), 0)
if __name__ == "__main__":
......
......@@ -278,6 +278,7 @@ class StoreHelper (Helper):
# not yet happened.
def reset_graph_updates_tracking (self):
self.class_to_track = None
self.inserts_list = []
self.deletes_list = []
self.inserts_match_function = None
......@@ -292,21 +293,26 @@ class StoreHelper (Helper):
"""
exit_loop = False
if inserts_list is not None:
if self.inserts_match_function is not None:
# The match function will remove matched entries from the list
(exit_loop, inserts_list) = self.inserts_match_function (inserts_list)
self.inserts_list += inserts_list
if class_name == self.class_to_track:
log("GraphUpdated for %s: %i deletes, %i inserts" % (class_name, len(deletes_list), len(inserts_list)))
if deletes_list is not None:
if self.deletes_match_function is not None:
(exit_loop, deletes_list) = self.deletes_match_function (deletes_list)
self.deletes_list += deletes_list
if inserts_list is not None:
if self.inserts_match_function is not None:
# The match function will remove matched entries from the list
(exit_loop, inserts_list) = self.inserts_match_function (inserts_list)
self.inserts_list += inserts_list
if exit_loop:
GLib.source_remove(self.graph_updated_timeout_id)
self.graph_updated_timeout_id = 0
self.loop.quit ()
if deletes_list is not None:
if self.deletes_match_function is not None:
(exit_loop, deletes_list) = self.deletes_match_function (deletes_list)
self.deletes_list += deletes_list
if exit_loop:
GLib.source_remove(self.graph_updated_timeout_id)
self.graph_updated_timeout_id = 0
self.loop.quit ()
else:
log("Ignoring GraphUpdated for class %s, currently tracking %s" % (class_name, self.class_to_track))
def _enable_await_timeout (self):
self.graph_updated_timeout_id = GLib.timeout_add_seconds (REASONABLE_TIMEOUT,
......@@ -317,6 +323,9 @@ class StoreHelper (Helper):
Block until a resource matching the parameters becomes available
"""
assert (self.inserts_match_function == None)
assert (self.class_to_track == None)
self.class_to_track = rdf_class
self.matched_resource_urn = None
self.matched_resource_id = None
......@@ -339,7 +348,7 @@ class StoreHelper (Helper):
id = insert[1]
if not matched_creation:
where = " ?urn a %s " % rdf_class
where = " ?urn a <%s> " % rdf_class
if url is not None:
where += "; nie:url \"%s\"" % url
......@@ -390,14 +399,16 @@ class StoreHelper (Helper):
raise GraphUpdateTimeoutException("Timeout waiting for resource: class %s, URL %s, title %s" % (rdf_class, url, title))
self.inserts_match_function = None
self.class_to_track = None
return (self.matched_resource_id, self.matched_resource_urn)
def await_resource_deleted (self, id):
def await_resource_deleted (self, rdf_class, id):
"""
Block until we are notified of a resources deletion
"""
assert (self.deletes_match_function == None)
assert (self.class_to_track == None)
def find_resource_deletion (deletes_list):
log ("find_resource_deletion: looking for %i in %s" % (id, deletes_list))
......@@ -431,14 +442,21 @@ class StoreHelper (Helper):
except GraphUpdateTimeoutException:
raise GraphUpdateTimeoutException ("Resource %i has not been deleted." % id)
self.deletes_match_function = None
self.class_to_track = None
return
def await_property_changed (self, subject_id, property_uri):
def await_property_changed (self, rdf_class, subject_id, property_uri):
"""
Block until a property of a resource is updated or inserted.
"""
assert (self.inserts_match_function == None)
assert (self.deletes_match_function == None)
assert (self.class_to_track == None)
log ("Await change to %i %s (%i, %i existing)" % (subject_id, property_uri, len(self.inserts_list), len(self.deletes_list)))
self.class_to_track = rdf_class
property_id = self.get_resource_id_by_uri(property_uri)
......@@ -471,8 +489,9 @@ class StoreHelper (Helper):
self.loop.run_checked ()
except GraphUpdateTimeoutException:
raise GraphUpdateTimeoutException(
"Timeout waiting for property change, subject %i property %s" % (subject_id, property_uri))
"Timeout waiting for property change, subject %i property %s (%i)" % (subject_id, property_uri, property_id))
self.inserts_match_function = None
self.class_to_track = None
def query (self, query, timeout=5000, **kwargs):
return self.resources.SparqlQuery ('(s)', query, timeout=timeout, **kwargs)
......
......@@ -30,8 +30,11 @@ import shutil
import tempfile
from itertools import chain
DEFAULT_TEXT = "Some stupid content, to have a test file"
NFO_TEXT_DOCUMENT = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#TextDocument'
def ensure_dir_exists(dirname):
if not os.path.exists(dirname):
......@@ -114,8 +117,7 @@ class CommonTrackerMinerTest (ut.TestCase):
f.write (DEFAULT_TEXT)
for tf in monitored_files:
self.tracker.await_resource_inserted(
'nfo:TextDocument', url=self.uri(tf))
self.tracker.await_resource_inserted(NFO_TEXT_DOCUMENT, url=self.uri(tf))
def remove_test_data(self):
try:
......@@ -154,11 +156,11 @@ class CommonTrackerMinerFTSTest (CommonTrackerMinerTest):
if exists:
subject_id = self.tracker.get_resource_id(self.uri(self.testfile))
self.tracker.await_property_changed(
self.tracker.await_property_changed(NFO_TEXT_DOCUMENT,
subject_id=subject_id, property_uri='nie:plainTextContent')
else:
self.tracker.await_resource_inserted(
rdf_class='nfo:Document', url=self.uri(self.testfile),
rdf_class=NFO_TEXT_DOCUMENT, url=self.uri(self.testfile),
required_property='nie:plainTextContent')
self.tracker.reset_graph_updates_tracking()
......
......@@ -32,6 +32,8 @@ TEST_FILE_JPEG = "writeback-test-1.jpeg"
TEST_FILE_TIFF = "writeback-test-2.tif"
TEST_FILE_PNG = "writeback-test-4.png"
NFO_IMAGE = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#Image'
WRITEBACK_TMP_DIR = os.path.join (cfg.TEST_MONITORED_TMP_DIR, "writeback")
index_dirs = [WRITEBACK_TMP_DIR]
......@@ -92,7 +94,7 @@ class CommonTrackerWritebackTest (ut.TestCase):
# tracker-extract. The extractor adds nie:contentCreated for
# image resources, so know once this property is set the
# extraction is complete.
self.system.store.await_resource_inserted('nfo:Image', url=url, required_property='nfo:width')
self.system.store.await_resource_inserted(NFO_IMAGE, url=url, required_property='nfo:width')
await_resource_extraction (self.get_test_filename_jpeg())
await_resource_extraction (self.get_test_filename_tiff())
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment