Commit 4c5e508b authored by Thibault Saunier's avatar Thibault Saunier

Handle deleted proxy files when loading a project

We handle it as follow:
Say, the loading project as file A and its proxy A.proxy

 - In Project::missing-uri, return the proxy target URI so the proxy
   is, proxied by it target (A.proxy will be proxied by A)
 - As soon as the A asset is ready, we start creating its proxy
 - Once the A.proxy is created, we reload it, unproxy it (to avoid proxy
   cycles), and start using it as a proxy for A

Also fix several places where we were considering that an asset
with a ->proxy_target != None was a proxy in our terms, it is not true
anymore as during the time where we are recreating 'A.proxy',
A.props.proxy_target is actually A.proxy, but it is no a proxy for us at
that point (just a temporary redirection).

Fixes T7560
Reviewed-by: 's avatarAlex Băluț <&lt;alexandru.balut@gmail.com&gt;>
Differential Revision: https://phabricator.freedesktop.org/D1815
parent b9c61b39
......@@ -46,10 +46,10 @@ from pitivi.settings import GlobalSettings
from pitivi.timeline.previewers import ThumbnailCache
from pitivi.utils.loggable import Loggable
from pitivi.utils.misc import disconnectAllByFunc
from pitivi.utils.misc import get_proxy_target
from pitivi.utils.misc import path_from_uri
from pitivi.utils.misc import PathWalker
from pitivi.utils.misc import quote_uri
from pitivi.utils.proxy import get_proxy_target
from pitivi.utils.proxy import ProxyingStrategy
from pitivi.utils.proxy import ProxyManager
from pitivi.utils.ui import beautify_asset
......@@ -310,7 +310,8 @@ class AssetThumbnail(Loggable):
def __setState(self):
asset = self.__asset
target = asset.get_proxy_target()
if target and not target.get_error():
if self.proxy_manager.is_proxy_asset(asset) and target \
and not target.get_error():
# The asset is a proxy.
self.state = self.PROXIED
elif asset.proxying_error:
......@@ -809,17 +810,24 @@ class MediaLibraryWidget(Gtk.Box, Loggable):
def _assetLoadingProgressCb(self, project, progress, estimated_time):
self._progressbar.set_fraction(progress / 100)
proxying_files = []
for row in self.storemodel:
row[COL_INFOTEXT] = beautify_asset(row[COL_ASSET])
asset = row[COL_ASSET]
row[COL_INFOTEXT] = beautify_asset(asset)
if not asset.ready:
proxying_files.append(asset)
if row[COL_THUMB_DECORATOR].state != AssetThumbnail.IN_PROGRESS:
thumbs_decorator = AssetThumbnail(asset, self.app.proxy_manager)
row[COL_ICON_64] = thumbs_decorator.small_thumb
row[COL_ICON_128] = thumbs_decorator.large_thumb
row[COL_THUMB_DECORATOR] = thumbs_decorator
if progress == 0:
self._startImporting(project)
return
if project.loaded:
proxying_files = [asset
for asset in project.loading_assets
if not asset.ready]
if estimated_time:
self.__last_proxying_estimate_time = beautify_ETA(int(
estimated_time * Gst.SECOND))
......@@ -844,7 +852,14 @@ class MediaLibraryWidget(Gtk.Box, Loggable):
self._doneImporting()
def __assetProxyingCb(self, proxy, unused_pspec):
self.debug("Proxy is %s", proxy.props.id)
if not self.app.proxy_manager.is_proxy_asset(proxy):
self.info("Proxy is not a proxy in our terms (handling deleted proxy"
" files while loading a project?) - ignore it")
return
self.debug("Proxy is %s - %s", proxy.props.id,
proxy.get_proxy_target())
self.__removeAsset(proxy)
if proxy.get_proxy_target() is not None:
......@@ -1174,7 +1189,7 @@ class MediaLibraryWidget(Gtk.Box, Loggable):
menu_model.append(text, "assets.%s" % action.get_name().replace(" ", "."))
proxies = [asset.get_proxy_target() for asset in assets
if asset.get_proxy_target()]
if self.app.proxy_manager.is_proxy_asset(asset)]
in_progress = [asset.creation_progress for asset in assets
if asset.creation_progress < 100]
......
......@@ -687,6 +687,17 @@ class Project(Loggable, GES.Project):
self.nb_remaining_file_to_import = 0
self.nb_imported_files = 0
# Main assets that were proxied when saving the project but
# whose proxies had been deleted from the filesystem. The
# proxy files are being regenerated.
self.__deleted_proxy_files = set()
# List of proxy assets uris that were deleted on the filesystem
# and we are waiting for the main asset (ie. the file from
# which the proxy was generated) to be loaded before we can try to
# regenerate the proxy.
self.__awaited_deleted_proxy_targets = set()
# Project property default values
self.register_meta(GES.MetaFlag.READWRITE, "name", name)
self.register_meta(GES.MetaFlag.READWRITE, "author", "")
......@@ -980,6 +991,10 @@ class Project(Loggable, GES.Project):
def __get_loading_project_progress(self):
"""Computes current advancement of asset loading during project loading.
During project loading we keep all loading assets to keep track of real advancement
during the whole process, whereas while adding new assets, they get removed from
the `loading_assets` list once the proxy is ready.
Returns:
int: The current asset loading progress (in percent).
"""
......@@ -989,7 +1004,11 @@ class Project(Loggable, GES.Project):
if asset.creation_progress < 100:
all_ready = False
else:
asset.ready = True
# Check that we are not recreating deleted proxy
proxy_uri = self.app.proxy_manager.getProxyUri(asset)
if proxy_uri and proxy_uri not in self.__deleted_proxy_files and \
asset.props.id not in self.__awaited_deleted_proxy_targets:
asset.ready = True
num_loaded += 1
if all_ready:
......@@ -1077,10 +1096,16 @@ class Project(Loggable, GES.Project):
self.__updateAssetLoadingProgress()
def __proxyReadyCb(self, unused_proxy_manager, asset, proxy):
if proxy and proxy.props.id in self.__deleted_proxy_files:
self.info("Recreated proxy is now ready, stop having"
" its target as a proxy.")
proxy.unproxy(asset)
self.__setProxy(asset, proxy)
def __setProxy(self, asset, proxy):
asset.creation_progress = 100
asset.ready = True
if proxy:
proxy.ready = False
proxy.error = None
......@@ -1108,6 +1133,33 @@ class Project(Loggable, GES.Project):
self._prepare_asset_processing(asset)
def __regenerate_missing_proxy(self, asset):
self.info("Re generating deleted proxy file %s.", asset.props.id)
GES.Asset.needs_reload(GES.UriClip, asset.props.id)
self._prepare_asset_processing(asset)
asset.force_proxying = True
self.app.proxy_manager.add_job(asset)
self.__updateAssetLoadingProgress()
def do_missing_uri(self, error, asset):
if self.app.proxy_manager.is_proxy_asset(asset):
self.debug("Missing proxy file: %s", asset.props.id)
target_uri = self.app.proxy_manager.getTargetUri(asset)
GES.Asset.needs_reload(GES.UriClip, asset.props.id)
# Check if the target has already been loaded.
target = [asset for asset in self.list_assets(GES.UriClip) if
asset.props.id == target_uri]
if target:
self.__regenerate_missing_proxy(target[0])
else:
self.__awaited_deleted_proxy_targets.add(target_uri)
self.__deleted_proxy_files.add(asset.props.id)
return target_uri
return GES.Project.do_missing_uri(self, error, asset)
def _prepare_asset_processing(self, asset):
asset.creation_progress = 0
asset.error = None
......@@ -1138,12 +1190,19 @@ class Project(Loggable, GES.Project):
" it must not be proxied", asset.get_id())
return
if asset.props.id in self.__awaited_deleted_proxy_targets:
self.__regenerate_missing_proxy(asset)
self.__awaited_deleted_proxy_targets.remove(asset.props.id)
elif asset.props.id in self.__deleted_proxy_files:
self.info("Deleted proxy file %s now ready again.", asset.props.id)
self.__deleted_proxy_files.remove(asset.props.id)
if self.loaded:
if not asset.get_proxy_target() in self.list_assets(GES.Extractable):
self.app.proxy_manager.add_job(asset)
else:
self.debug("Project still loading, not using proxies: %s",
asset.props.id)
asset.props.id)
asset.creation_progress = 100
self.__updateAssetLoadingProgress()
......@@ -1170,6 +1229,15 @@ class Project(Loggable, GES.Project):
self.ges_timeline.props.auto_transition = True
self._ensureLayer()
if self.uri:
self.loading_assets = set([asset for asset in self.loading_assets if
self.app.proxy_manager.is_asset_queued(asset)])
if self.loading_assets:
self.debug("The following assets are still being transcoded: %s."
" (They must be proxied assets with missing/deleted"
" proxy files).", self.loading_assets)
if self.scenario is not None:
return
......@@ -1250,7 +1318,13 @@ class Project(Loggable, GES.Project):
def use_proxies_for_assets(self, assets):
originals = []
for asset in assets:
if not asset.get_proxy_target():
if not self.app.proxy_manager.is_proxy_asset(asset):
target = asset.get_proxy_target()
if target and target.props.id == self.app.proxy_manager.getProxyUri(asset):
self.info("Missing proxy needs to be recreated after cancelling"
" its recreation")
target.unproxy(asset)
# The asset is not a proxy.
originals.append(asset)
if originals:
......@@ -1264,8 +1338,8 @@ class Project(Loggable, GES.Project):
def disable_proxies_for_assets(self, assets, delete_proxy_file=False):
for asset in assets:
proxy_target = asset.get_proxy_target()
if proxy_target:
if self.app.proxy_manager.is_proxy_asset(asset):
proxy_target = asset.get_proxy_target()
# The asset is a proxy for the proxy_target original asset.
self.debug("Stop proxying %s", proxy_target.props.id)
proxy_target.set_proxy(None)
......
......@@ -36,12 +36,12 @@ from pitivi.settings import GlobalSettings
from pitivi.settings import xdg_cache_home
from pitivi.utils.loggable import Loggable
from pitivi.utils.misc import binary_search
from pitivi.utils.misc import get_proxy_target
from pitivi.utils.misc import hash_file
from pitivi.utils.misc import path_from_uri
from pitivi.utils.misc import quantize
from pitivi.utils.misc import quote_uri
from pitivi.utils.pipeline import MAX_BRINGING_TO_PAUSED_DURATION
from pitivi.utils.proxy import get_proxy_target
from pitivi.utils.system import CPUUsageTracker
from pitivi.utils.timeline import Zoomable
from pitivi.utils.ui import EXPANDED_SIZE
......
......@@ -82,21 +82,6 @@ def call_false(function, *args, **kwargs):
return False
def get_proxy_target(obj):
if isinstance(obj, GES.UriClip):
asset = obj.get_asset()
elif isinstance(obj, GES.TrackElement):
asset = obj.get_parent().get_asset()
else:
asset = obj
target = asset.get_proxy_target()
if target and target.get_error() is None:
asset = target
return asset
# ------------------------------ URI helpers --------------------------------
def isWritable(path):
......
......@@ -235,9 +235,15 @@ class ProxyManager(GObject.Object, Loggable):
<filename>.<file_size>.<proxy_extension>
"""
asset_file = Gio.File.new_for_uri(asset.get_id())
file_size = asset_file.query_info(Gio.FILE_ATTRIBUTE_STANDARD_SIZE,
Gio.FileQueryInfoFlags.NONE,
None).get_size()
try:
file_size = asset_file.query_info(Gio.FILE_ATTRIBUTE_STANDARD_SIZE,
Gio.FileQueryInfoFlags.NONE,
None).get_size()
except GLib.Error as err:
if err.matches(Gio.io_error_quark(), Gio.IOErrorEnum.NOT_FOUND):
return None
else:
raise
return "%s.%s.%s" % (asset.get_id(), file_size, self.proxy_extension)
......@@ -358,6 +364,10 @@ class ProxyManager(GObject.Object, Loggable):
self.emit("progress", asset, asset.creation_progress, estimated_time)
def __proxyingPositionChangedCb(self, transcoder, position, asset):
if transcoder not in self.__running_transcoders:
self.info("Position changed after job cancelled!")
return
self._transcoded_durations[asset] = position / Gst.SECOND
duration = transcoder.props.duration
......@@ -481,3 +491,19 @@ class ProxyManager(GObject.Object, Loggable):
force_proxying)
self.__createTranscoder(asset)
return
def get_proxy_target(obj):
if isinstance(obj, GES.UriClip):
asset = obj.get_asset()
elif isinstance(obj, GES.TrackElement):
asset = obj.get_parent().get_asset()
else:
asset = obj
if ProxyManager.is_proxy_asset(asset):
target = asset.get_proxy_target()
if target and target.get_error() is None:
asset = target
return asset
......@@ -44,8 +44,8 @@ from pitivi.configure import get_pixmap_dir
from pitivi.utils.loggable import doLog
from pitivi.utils.loggable import ERROR
from pitivi.utils.loggable import INFO
from pitivi.utils.misc import get_proxy_target
from pitivi.utils.misc import path_from_uri
from pitivi.utils.proxy import get_proxy_target
# Dimensions in pixels
......
......@@ -27,9 +27,11 @@ from unittest import TestCase
from gi.repository import GES
from gi.repository import Gst
from pitivi import medialibrary
from pitivi.project import Project
from pitivi.project import ProjectManager
from pitivi.utils.misc import path_from_uri
from pitivi.utils.proxy import ProxyingStrategy
from tests import common
......@@ -342,6 +344,134 @@ class TestProjectLoading(common.TestCase):
self.assertEqual(len(assets), 1, assets)
def load_project_with_missing_proxy(self):
"""Loads a project with missing proxies."""
uris = [common.get_sample_uri("1sec_simpsons_trailer.mp4")]
proxy_uri = uris[0] + ".232417.proxy.mkv"
PROJECT_STR = """<ges version='0.3'>
<project properties='properties;' metadatas='metadatas, name=(string)&quot;New\ Project&quot;, author=(string)Unknown, render-scale=(double)100;'>
<encoding-profiles>
</encoding-profiles>
<ressources>
<asset id='%(uri)s' extractable-type-name='GESUriClip' properties='properties, supported-formats=(int)6, duration=(guint64)1228000000;' metadatas='metadatas, audio-codec=(string)&quot;MPEG-4\ AAC\ audio&quot;, maximum-bitrate=(uint)130625, bitrate=(uint)130625, datetime=(datetime)2007-02-19T05:03:04Z, encoder=(string)Lavf54.6.100, container-format=(string)&quot;ISO\ MP4/M4A&quot;, video-codec=(string)&quot;H.264\ /\ AVC&quot;, file-size=(guint64)232417;' proxy-id='file:///home/thiblahute/devel/pitivi/flatpak/pitivi/tests/samples/1sec_simpsons_trailer.mp4.232417.proxy.mkv' />
<asset id='%(proxy_uri)s' extractable-type-name='GESUriClip' properties='properties, supported-formats=(int)6, duration=(guint64)1228020833;' metadatas='metadatas, container-format=(string)Matroska, audio-codec=(string)Opus, language-code=(string)en, encoder=(string)Lavf54.6.100, bitrate=(uint)64000, video-codec=(string)&quot;Motion\ JPEG&quot;, file-size=(guint64)4695434;' />
</ressources>
<timeline properties='properties, auto-transition=(boolean)true, snapping-distance=(guint64)0;' metadatas='metadatas, duration=(guint64)0;'>
<track caps='video/x-raw(ANY)' track-type='4' track-id='0' properties='properties, async-handling=(boolean)false, message-forward=(boolean)true, caps=(string)&quot;video/x-raw\(ANY\)&quot;, restriction-caps=(string)&quot;video/x-raw\,\ width\=\(int\)720\,\ height\=\(int\)576\,\ framerate\=\(fraction\)25/1&quot;, mixing=(boolean)true;' metadatas='metadatas;'/>
<track caps='audio/x-raw(ANY)' track-type='2' track-id='1' properties='properties, async-handling=(boolean)false, message-forward=(boolean)true, caps=(string)&quot;audio/x-raw\(ANY\)&quot;, restriction-caps=(string)&quot;audio/x-raw\,\ format\=\(string\)S32LE\,\ channels\=\(int\)2\,\ rate\=\(int\)44100\,\ layout\=\(string\)interleaved&quot;, mixing=(boolean)true;' metadatas='metadatas;'/>
<layer priority='0' properties='properties, auto-transition=(boolean)true;' metadatas='metadatas, volume=(float)1;'>
<clip id='0' asset-id='%(proxy_uri)s' type-name='GESUriClip' layer-priority='0' track-types='6' start='0' duration='1228000000' inpoint='0' rate='0' properties='properties, name=(string)uriclip0, mute=(boolean)false, is-image=(boolean)false;' >
<source track-id='1' children-properties='properties, GstVolume::mute=(boolean)false, GstVolume::volume=(double)1;'>
<binding type='direct' source_type='interpolation' property='volume' mode='1' track_id='1' values =' 0:0.10000000000000001 1228000000:0.10000000000000001 '/>
</source>
<source track-id='0' children-properties='properties, GstFramePositioner::alpha=(double)1, GstDeinterlace::fields=(int)0, GstFramePositioner::height=(int)720, GstDeinterlace::mode=(int)0, GstFramePositioner::posx=(int)0, GstFramePositioner::posy=(int)0, GstDeinterlace::tff=(int)0, GstFramePositioner::width=(int)1280;'>
<binding type='direct' source_type='interpolation' property='alpha' mode='1' track_id='0' values =' 0:1 1228000000:1 '/>
</source>
</clip>
</layer>
<groups>
</groups>
</timeline>
</project>
</ges>""" % {"uri": uris[0], "proxy_uri": proxy_uri}
app = common.create_pitivi(proxyingStrategy=ProxyingStrategy.ALL)
proxy_manager = app.proxy_manager
project_manager = app.project_manager
mainloop = common.create_main_loop()
unused, xges_path = tempfile.mkstemp(suffix=".xges")
proj_uri = "file://" + os.path.abspath(xges_path)
app.project_manager.saveProject(uri=proj_uri)
medialib = medialibrary.MediaLibraryWidget(app)
with open(proj_uri[len("file://"):], "w") as f:
f.write(PROJECT_STR)
# Remove proxy
common.clean_proxy_samples()
def closing_project_cb(*args, **kwargs):
# Do not ask whether to save project on closing.
return True
def proxy_ready_cb(proxy_manager, asset, proxy):
self.assertEqual(proxy.props.id, proxy_uri)
mainloop.quit()
project_manager.connect("closing-project", closing_project_cb)
proxy_manager.connect_after("proxy-ready", proxy_ready_cb)
app.project_manager.loadProject(proj_uri)
return mainloop, app, medialib, proxy_uri
def test_load_project_with_missing_proxy(self):
"""Checks loading a project with missing proxies."""
mainloop, app, medialib, proxy_uri = self.load_project_with_missing_proxy()
mainloop.run()
self.assertEqual(len(medialib.storemodel), 1)
self.assertEqual(medialib.storemodel[0][medialibrary.COL_ASSET].props.id,
proxy_uri)
self.assertEqual(medialib.storemodel[0][medialibrary.COL_THUMB_DECORATOR].state,
medialibrary.AssetThumbnail.PROXIED)
def test_load_project_with_missing_proxy_progress_tracking(self):
"""Checks progress tracking of loading project with missing proxies."""
from gi.repository import GstTranscoder
# Disable proxy generation by not making it start ever.
# This way we are sure it will not finish before we test
# the state while it is being rebuilt.
with mock.patch.object(GstTranscoder.Transcoder, "run_async"):
mainloop, app, medialib, proxy_uri = self.load_project_with_missing_proxy()
uri = common.get_sample_uri("1sec_simpsons_trailer.mp4")
app.project_manager.connect("new-project-loaded", lambda x, y: mainloop.quit())
mainloop.run()
self.assertEqual(len(medialib.storemodel), 1)
self.assertEqual(medialib.storemodel[0][medialibrary.COL_ASSET].props.id,
uri)
self.assertEqual(medialib.storemodel[0][medialibrary.COL_THUMB_DECORATOR].state,
medialibrary.AssetThumbnail.IN_PROGRESS)
def test_load_project_with_missing_proxy_stop_generating_and_proxy(self):
"""Checks cancelling creation of a missing proxies and forcing it again."""
from gi.repository import GstTranscoder
# Disable proxy generation by not making it start ever.
# This way we are sure it will not finish before we test
# stop generating the proxy and restart it.
with mock.patch.object(GstTranscoder.Transcoder, "run_async"):
mainloop, app, medialib, proxy_uri = self.load_project_with_missing_proxy()
uri = common.get_sample_uri("1sec_simpsons_trailer.mp4")
app.project_manager.connect("new-project-loaded", lambda x, y: mainloop.quit())
mainloop.run()
asset = medialib.storemodel[0][medialibrary.COL_ASSET]
app.project_manager.current_project.disable_proxies_for_assets([asset])
row, = medialib.storemodel
asset = row[medialibrary.COL_ASSET]
self.assertEqual(medialib._progressbar.get_fraction(), 1.0)
self.assertEqual(asset.props.id, uri)
self.assertEqual(asset.ready, True)
self.assertEqual(asset.creation_progress, 100)
self.assertEqual(row[medialibrary.COL_THUMB_DECORATOR].state,
medialibrary.AssetThumbnail.NO_PROXY)
app.project_manager.current_project.use_proxies_for_assets([asset])
mainloop.run()
row, = medialib.storemodel
asset = row[medialibrary.COL_ASSET]
self.assertEqual(medialib._progressbar.is_visible(), False)
self.assertEqual(asset.props.id, proxy_uri)
self.assertEqual(asset.ready, True)
self.assertEqual(asset.creation_progress, 100)
self.assertEqual(row[medialibrary.COL_THUMB_DECORATOR].state,
medialibrary.AssetThumbnail.PROXIED)
class TestProjectSettings(common.TestCase):
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment