Commit 690523f4 authored by Sam Thursfield's avatar Sam Thursfield

functional-tests: Rationalize handling of temporary directories

Tests were sharing a test directory, which caused issues when they were
run in parallel by Meson.

As a bonus, we now clean up the ~/tracker-tests directory on successful
test runs.
parent eb5d5071
......@@ -94,8 +94,8 @@ class MinerCrawlTest (CommonTrackerMinerTest):
Copy an file from unmonitored directory to monitored directory
and verify if data base is updated accordingly
"""
source = os.path.join (self.datadir, "test-no-monitored", "file0.txt")
dest = os.path.join (self.datadir, "test-monitored", "file0.txt")
source = os.path.join (self.workdir, "test-no-monitored", "file0.txt")
dest = os.path.join (self.workdir, "test-monitored", "file0.txt")
shutil.copyfile (source, dest)
dest_id, dest_urn = self.system.store.await_resource_inserted (NFO_DOCUMENT, self.uri(dest))
......@@ -121,8 +121,8 @@ class MinerCrawlTest (CommonTrackerMinerTest):
"""
# Copy from monitored to unmonitored
source = os.path.join (self.datadir, "test-monitored", "file1.txt")
dest = os.path.join (self.datadir, "test-no-monitored", "file1.txt")
source = os.path.join (self.workdir, "test-monitored", "file1.txt")
dest = os.path.join (self.workdir, "test-no-monitored", "file1.txt")
shutil.copyfile (source, dest)
time.sleep (1)
......@@ -141,8 +141,8 @@ class MinerCrawlTest (CommonTrackerMinerTest):
"""
Copy a file between monitored directories
"""
source = os.path.join (self.datadir, "test-monitored", "file1.txt")
dest = os.path.join (self.datadir, "test-monitored", "dir1", "dir2", "file-test04.txt")
source = os.path.join (self.workdir, "test-monitored", "file1.txt")
dest = os.path.join (self.workdir, "test-monitored", "dir1", "dir2", "file-test04.txt")
shutil.copyfile (source, dest)
dest_id, dest_urn = self.system.store.await_resource_inserted (NFO_DOCUMENT, self.uri(dest))
......@@ -165,8 +165,8 @@ class MinerCrawlTest (CommonTrackerMinerTest):
"""
Move a file from unmonitored to monitored directory
"""
source = os.path.join (self.datadir, "test-no-monitored", "file0.txt")
dest = os.path.join (self.datadir, "test-monitored", "dir1", "file-test05.txt")
source = os.path.join (self.workdir, "test-no-monitored", "file0.txt")
dest = os.path.join (self.workdir, "test-monitored", "dir1", "file-test05.txt")
shutil.move (source, dest)
dest_id, dest_urn = self.system.store.await_resource_inserted (NFO_DOCUMENT, self.uri(dest))
......
......@@ -47,18 +47,9 @@ VALID_FILE_TITLE = 'Simply Juvenile'
TRACKER_EXTRACT_FAILURE_DATA_SOURCE = 'tracker:extractor-failure-data-source'
def ensure_dir_exists(dirname):
if not os.path.exists(dirname):
os.makedirs(dirname)
class ExtractorDecoratorTest(ut.TestCase):
def setUp(self):
ensure_dir_exists(cfg.TEST_MONITORED_TMP_DIR)
# It's important that this directory is NOT inside /tmp, because
# monitoring files in /tmp usually doesn't work.
self.datadir = tempfile.mkdtemp(dir=cfg.TEST_MONITORED_TMP_DIR)
self.datadir = cfg.create_monitored_test_dir()
config = {
cfg.DCONF_MINER_SCHEMA: {
......@@ -76,9 +67,9 @@ class ExtractorDecoratorTest(ut.TestCase):
self.system.tracker_miner_fs_testing_start()
def tearDown(self):
self.system.tracker_miner_fs_testing_stop()
self.system.finish()
shutil.rmtree(self.datadir)
cfg.remove_monitored_test_dir(self.datadir)
def test_reextraction(self):
"""Tests whether known files are still re-extracted on user request."""
......@@ -110,6 +101,8 @@ class ExtractorDecoratorTest(ut.TestCase):
assert len(title_result) == 1
self.assertEqual(title_result[0][0], VALID_FILE_TITLE)
os.remove(file_path)
if __name__ == '__main__':
ut.main()
......@@ -28,18 +28,6 @@ import shutil
import os
import time
APPLICATIONS_TMP_DIR = os.path.join (cfg.TEST_MONITORED_TMP_DIR, "test-applications-monitored")
index_dirs = [APPLICATIONS_TMP_DIR]
CONF_OPTIONS = {
cfg.DCONF_MINER_SCHEMA: {
'index-recursive-directories': GLib.Variant.new_strv(index_dirs),
'index-single-directories': GLib.Variant.new_strv([]),
'index-optical-discs': GLib.Variant.new_boolean(False),
'index-removable-devices': GLib.Variant.new_boolean(False),
}
}
# Copy rate, 10KBps (1024b/100ms)
SLOWCOPY_RATE = 1024
......@@ -68,7 +56,7 @@ class CommonTrackerApplicationTest (ut.TestCase):
return self.datadir
def get_dest_dir (self):
return APPLICATIONS_TMP_DIR
return self.workdir
def slowcopy_file_fd (self, src, fdest, rate=SLOWCOPY_RATE):
"""
......@@ -94,10 +82,18 @@ class CommonTrackerApplicationTest (ut.TestCase):
@classmethod
def setUp (self):
# Create temp directory to monitor
if (os.path.exists (APPLICATIONS_TMP_DIR)):
shutil.rmtree (APPLICATIONS_TMP_DIR)
os.makedirs (APPLICATIONS_TMP_DIR)
self.workdir = cfg.create_monitored_test_dir()
index_dirs = [self.workdir]
CONF_OPTIONS = {
cfg.DCONF_MINER_SCHEMA: {
'index-recursive-directories': GLib.Variant.new_strv(index_dirs),
'index-single-directories': GLib.Variant.new_strv([]),
'index-optical-discs': GLib.Variant.new_boolean(False),
'index-removable-devices': GLib.Variant.new_boolean(False),
}
}
# Use local directory if available. Installation otherwise.
if os.path.exists (os.path.join (os.getcwd (),
......@@ -109,20 +105,12 @@ class CommonTrackerApplicationTest (ut.TestCase):
"tracker-tests",
"test-apps-data")
self.system = TrackerSystemAbstraction ()
self.system.tracker_all_testing_start (CONF_OPTIONS)
# Returns when ready
self.tracker = self.system.store
log ("Ready to go!")
@classmethod
def tearDown (self):
#print "Stopping the daemon in test mode (Doing nothing now)"
self.system.tracker_all_testing_stop ()
self.system.finish ()
# Remove monitored directory
if (os.path.exists (APPLICATIONS_TMP_DIR)):
shutil.rmtree (APPLICATIONS_TMP_DIR)
cfg.remove_monitored_test_dir(self.workdir)
......@@ -20,8 +20,11 @@
"Constants describing Tracker D-Bus services"
import errno
import json
import os
import tempfile
if 'TRACKER_FUNCTIONAL_TEST_CONFIG' not in os.environ:
raise RuntimeError("The TRACKER_FUNCTIONAL_TEST_CONFIG environment "
......@@ -86,22 +89,42 @@ TRACKER_WRITEBACK_PATH = os.path.normpath(expandvars(config['TRACKER_WRITEBACK_P
DATADIR = os.path.normpath(expandvars(config['RAW_DATAROOT_DIR']))
TEST_TMP_DIR = os.path.join (os.environ["HOME"], "tracker-tests")
def generated_ttl_dir():
return os.path.join(TOP_BUILD_DIR, 'tests', 'functional-tests', 'ttl')
TEST_MONITORED_TMP_DIR = TEST_TMP_DIR
if TEST_TMP_DIR.startswith('/tmp'):
# This path is used for test data for tests which expect filesystem monitoring
# to work. For this reason we must avoid it being on a tmpfs filesystem. Note
# that this MUST NOT be a hidden directory, as Tracker is hardcoded to ignore
# those. The 'ignore-files' configuration option can be changed, but the
# 'filter-hidden' property of TrackerIndexingTree is hardwired to be True at
# present :/
_TEST_MONITORED_TMP_DIR = os.path.join (os.environ["HOME"], "tracker-tests")
if _TEST_MONITORED_TMP_DIR.startswith('/tmp'):
if os.environ.has_key('REAL_HOME'):
# Note that this MUST NOT be a hidden directory, as Tracker is
# hardcoded to ignore those. The 'ignore-files' configuration option
# can be changed, but the 'filter-hidden' property of
# TrackerIndexingTree is hardwired to be True at present :/
TEST_MONITORED_TMP_DIR = os.path.join (os.environ["REAL_HOME"], "tracker-tests")
_TEST_MONITORED_TMP_DIR = os.path.join (os.environ["REAL_HOME"], "tracker-tests")
else:
print ("HOME is in the /tmp prefix - this will cause tests that rely " +
"on filesystem monitoring to fail as changes in that prefix are " +
"ignored.")
def generated_ttl_dir():
return os.path.join(TOP_BUILD_DIR, 'tests', 'functional-tests', 'ttl')
def create_monitored_test_dir():
'''Returns a unique tmpdir which supports filesystem monitor events.'''
if not os.path.exists(_TEST_MONITORED_TMP_DIR):
os.makedirs(_TEST_MONITORED_TMP_DIR)
return tempfile.mkdtemp(dir=_TEST_MONITORED_TMP_DIR)
def remove_monitored_test_dir(path):
# This will fail if the directory is not empty.
os.rmdir(path)
# We delete the parent directory if possible, to avoid cluttering the user's
# home dir, but there may be other tests running in parallel so we ignore
# an error if there are still files present in it.
try:
os.rmdir(_TEST_MONITORED_TMP_DIR)
except OSError as e:
if e.errno == errno.ENOTEMPTY:
pass
......@@ -43,13 +43,9 @@ def ensure_dir_exists(dirname):
class CommonTrackerMinerTest (ut.TestCase):
def setUp (self):
ensure_dir_exists(cfg.TEST_MONITORED_TMP_DIR)
self.workdir = cfg.create_monitored_test_dir()
# It's important that this directory is NOT inside /tmp, because
# monitoring files in /tmp usually doesn't work.
self.datadir = tempfile.mkdtemp(dir=cfg.TEST_MONITORED_TMP_DIR)
self.indexed_dir = os.path.join(self.datadir, 'test-monitored')
self.indexed_dir = os.path.join(self.workdir, 'test-monitored')
# It's important that this directory exists BEFORE we start Tracker:
# it won't monitor an indexing root for changes if it doesn't exist,
......@@ -90,14 +86,15 @@ class CommonTrackerMinerTest (ut.TestCase):
raise
def tearDown (self):
self.system.finish ()
self.remove_test_data ()
self.system.tracker_miner_fs_testing_stop ()
cfg.remove_monitored_test_dir(self.workdir)
def path (self, filename):
return os.path.join (self.datadir, filename)
return os.path.join (self.workdir, filename)
def uri (self, filename):
return "file://" + os.path.join (self.datadir, filename)
return "file://" + os.path.join (self.workdir, filename)
def create_test_data (self):
monitored_files = [
......@@ -121,7 +118,8 @@ class CommonTrackerMinerTest (ut.TestCase):
def remove_test_data(self):
try:
shutil.rmtree(self.datadir)
shutil.rmtree(os.path.join(self.workdir, 'test-monitored'))
shutil.rmtree(os.path.join(self.workdir, 'test-no-monitored'))
except Exception as e:
log("Failed to remove temporary data dir: %s" % e)
......
......@@ -2,6 +2,7 @@
import os
import subprocess
import shutil
import tempfile
import configuration as cfg
from gi.repository import GObject
......@@ -13,17 +14,8 @@ from dconf import DConfClient
import helpers
# Add this after fixing the backup/restore and ontology changes tests
#"G_DEBUG" : "fatal_criticals",
TEST_ENV_DIRS = { "XDG_DATA_HOME" : os.path.join (cfg.TEST_TMP_DIR, "data"),
"XDG_CACHE_HOME": os.path.join (cfg.TEST_TMP_DIR, "cache")}
TEST_ENV_VARS = { "LC_COLLATE": "en_GB.utf8" }
EXTRA_DIRS = [os.path.join (cfg.TEST_TMP_DIR, "data", "tracker"),
os.path.join (cfg.TEST_TMP_DIR, "cache", "tracker")]
REASONABLE_TIMEOUT = 5
class UnableToBootException (Exception):
......@@ -32,9 +24,22 @@ class UnableToBootException (Exception):
class TrackerSystemAbstraction (object):
def __init__(self, settings=None, ontodir=None):
self._basedir = None
self.extractor = None
self.miner_fs = None
self.store = None
self.writeback = None
self.set_up_environment (settings=settings, ontodir=ontodir)
self.store = None
def xdg_data_home(self):
return os.path.join(self._basedir, 'data')
def xdg_cache_home(self):
return os.path.join(self._basedir, 'cache')
def set_up_environment (self, settings=None, ontodir=None):
"""
Sets up the XDG_*_HOME variables and make sure the directories exist
......@@ -45,30 +50,28 @@ class TrackerSystemAbstraction (object):
GLib.Variant instance.
"""
helpers.log ("[Conf] Setting test environment...")
self._basedir = tempfile.mkdtemp()
for var, directory in TEST_ENV_DIRS.iteritems ():
helpers.log ("export %s=%s" %(var, directory))
self.__recreate_directory (directory)
os.environ [var] = directory
self._dirs = {
"XDG_DATA_HOME" : self.xdg_data_home(),
"XDG_CACHE_HOME": self.xdg_cache_home()
}
for directory in EXTRA_DIRS:
self.__recreate_directory (directory)
for var, directory in self._dirs.items():
os.makedirs (directory)
os.makedirs (os.path.join(directory, 'tracker'))
os.environ [var] = directory
if ontodir:
helpers.log ("export %s=%s" % ("TRACKER_DB_ONTOLOGIES_DIR", ontodir))
os.environ ["TRACKER_DB_ONTOLOGIES_DIR"] = ontodir
for var, value in TEST_ENV_VARS.iteritems ():
helpers.log ("export %s=%s" %(var, value))
os.environ [var] = value
# Previous loop should have set DCONF_PROFILE to the test location
if settings is not None:
self._apply_settings(settings)
helpers.log ("[Conf] environment ready")
def _apply_settings(self, settings):
for schema_name, contents in settings.iteritems():
dconf = DConfClient(schema_name)
......@@ -106,29 +109,29 @@ class TrackerSystemAbstraction (object):
raise UnableToBootException ("Unable to boot the store \n(" + str(e) + ")")
def tracker_store_prepare_journal_replay (self):
db_location = os.path.join (TEST_ENV_DIRS ['XDG_CACHE_HOME'], "tracker", "meta.db")
db_location = os.path.join (self.xdg_cache_home(), "tracker", "meta.db")
os.unlink (db_location)
lockfile = os.path.join (TEST_ENV_DIRS ['XDG_DATA_HOME'], "tracker", "data", ".ismeta.running")
lockfile = os.path.join (self.xdg_data_home(), "tracker", "data", ".ismeta.running")
f = open (lockfile, 'w')
f.write (" ")
f.close ()
def tracker_store_corrupt_dbs (self):
for filename in ["meta.db", "meta.db-wal"]:
db_path = os.path.join (TEST_ENV_DIRS ['XDG_CACHE_HOME'], "tracker", filename)
db_path = os.path.join (self.xdg_cache_home(), "tracker", filename)
f = open (db_path, "w")
for i in range (0, 100):
f.write ("Some stupid content... hohohoho, not a sqlite file anymore!\n")
f.close ()
def tracker_store_remove_journal (self):
db_location = os.path.join (TEST_ENV_DIRS ['XDG_DATA_HOME'], "tracker", "data")
db_location = os.path.join (self.xdg_data_home(), "tracker", "data")
shutil.rmtree (db_location)
os.mkdir (db_location)
def tracker_store_remove_dbs (self):
db_location = os.path.join (TEST_ENV_DIRS ['XDG_CACHE_HOME'], "tracker")
db_location = os.path.join (self.xdg_cache_home(), "tracker")
shutil.rmtree (db_location)
os.mkdir (db_location)
......@@ -184,7 +187,23 @@ class TrackerSystemAbstraction (object):
# This will stop all miner-fs, store and writeback
self.tracker_writeback_testing_stop ()
def __recreate_directory (self, directory):
if (os.path.exists (directory)):
shutil.rmtree (directory)
os.makedirs (directory)
def finish (self):
"""
Stop all running processes and remove all test data.
"""
if self.writeback:
self.writeback.stop ()
if self.extractor:
self.extractor.stop ()
if self.miner_fs:
self.miner_fs.stop ()
if self.store:
self.store.stop ()
for path in self._dirs.values():
shutil.rmtree(path)
os.rmdir(self._basedir)
......@@ -34,24 +34,6 @@ TEST_FILE_PNG = "writeback-test-4.png"
NFO_IMAGE = 'http://www.semanticdesktop.org/ontologies/2007/03/22/nfo#Image'
WRITEBACK_TMP_DIR = os.path.join (cfg.TEST_MONITORED_TMP_DIR, "writeback")
index_dirs = [WRITEBACK_TMP_DIR]
CONF_OPTIONS = {
cfg.DCONF_MINER_SCHEMA: {
'index-recursive-directories': GLib.Variant.new_strv(index_dirs),
'index-single-directories': GLib.Variant.new_strv([]),
'index-optical-discs': GLib.Variant.new_boolean(False),
'index-removable-devices': GLib.Variant.new_boolean(False),
},
'org.freedesktop.Tracker.Store': {
'graphupdated-delay': GLib.Variant.new_int32(100)
}
}
def uri (filename):
return "file://" + os.path.join (WRITEBACK_TMP_DIR, filename)
class CommonTrackerWritebackTest (ut.TestCase):
"""
......@@ -67,24 +49,32 @@ class CommonTrackerWritebackTest (ut.TestCase):
datadir = os.path.join (cfg.DATADIR, "tracker-tests",
"test-writeback-data")
if not os.path.exists(WRITEBACK_TMP_DIR):
os.makedirs(WRITEBACK_TMP_DIR)
else:
if not os.path.isdir(WRITEBACK_TMP_DIR):
raise Exception("%s exists already and is not a directory" % WRITEBACK_TMP_DIR)
for testfile in [TEST_FILE_JPEG, TEST_FILE_PNG,TEST_FILE_TIFF]:
origin = os.path.join (datadir, testfile)
log ("Copying %s -> %s" % (origin, WRITEBACK_TMP_DIR))
shutil.copy (origin, WRITEBACK_TMP_DIR)
log ("Copying %s -> %s" % (origin, self.workdir))
shutil.copy (origin, self.workdir)
def setUp(self):
#print "Starting the daemon in test mode"
self.workdir = cfg.create_monitored_test_dir()
index_dirs = [self.workdir]
CONF_OPTIONS = {
cfg.DCONF_MINER_SCHEMA: {
'index-recursive-directories': GLib.Variant.new_strv(index_dirs),
'index-single-directories': GLib.Variant.new_strv([]),
'index-optical-discs': GLib.Variant.new_boolean(False),
'index-removable-devices': GLib.Variant.new_boolean(False),
},
'org.freedesktop.Tracker.Store': {
'graphupdated-delay': GLib.Variant.new_int32(100)
}
}
self.__prepare_directories ()
self.system = TrackerSystemAbstraction ()
self.system = TrackerSystemAbstraction ()
self.system.tracker_writeback_testing_start (CONF_OPTIONS)
def await_resource_extraction(url):
......@@ -101,25 +91,25 @@ class CommonTrackerWritebackTest (ut.TestCase):
self.tracker = self.system.store
self.extractor = self.system.extractor
# Returns when ready
log ("Ready to go!")
def tearDown (self):
#print "Stopping the daemon in test mode (Doing nothing now)"
self.system.tracker_writeback_testing_stop ()
log ("Test finished")
self.system.finish ()
for testfile in [TEST_FILE_JPEG, TEST_FILE_PNG,TEST_FILE_TIFF]:
os.remove(os.path.join(self.workdir, testfile))
cfg.remove_monitored_test_dir(self.workdir)
def uri (self, filename):
return "file://" + os.path.join (self.workdir, filename)
@staticmethod
def get_test_filename_jpeg ():
return uri (TEST_FILE_JPEG)
def get_test_filename_jpeg (self):
return self.uri (TEST_FILE_JPEG)
@staticmethod
def get_test_filename_tiff ():
return uri (TEST_FILE_TIFF)
def get_test_filename_tiff (self):
return self.uri (TEST_FILE_TIFF)
@staticmethod
def get_test_filename_png ():
return uri (TEST_FILE_PNG)
def get_test_filename_png (self):
return self.uri (TEST_FILE_PNG)
def get_mtime (self, filename):
return os.stat(filename).st_mtime
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment