previewers.py 41.5 KB
Newer Older
1
# -*- coding: utf-8 -*-
2
# Pitivi video editor
3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18
# Copyright (c) 2013, Daniel Thul <daniel.thul@gmail.com>
#
# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 2.1 of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public
# License along with this program; if not, write to the
# Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
# Boston, MA 02110-1301, USA.
19
"""Previewers for the timeline."""
20
import contextlib
21
import os
22
import random
23 24
import sqlite3

25 26
import cairo
import numpy
27 28
from gi.repository import Gdk
from gi.repository import GdkPixbuf
29 30
from gi.repository import GES
from gi.repository import GLib
31
from gi.repository import GObject
32
from gi.repository import Gst
33
from gi.repository import Gtk
34

35
from pitivi.settings import get_dir
36
from pitivi.settings import GlobalSettings
37 38 39
from pitivi.settings import xdg_cache_home
from pitivi.utils.loggable import Loggable
from pitivi.utils.misc import hash_file
40
from pitivi.utils.misc import path_from_uri
41 42
from pitivi.utils.misc import quantize
from pitivi.utils.misc import quote_uri
43
from pitivi.utils.pipeline import MAX_BRINGING_TO_PAUSED_DURATION
44
from pitivi.utils.proxy import get_proxy_target
45 46 47 48
from pitivi.utils.system import CPUUsageTracker
from pitivi.utils.timeline import Zoomable
from pitivi.utils.ui import EXPANDED_SIZE

49
# Our C module optimizing waveforms rendering
50 51 52 53 54
try:
    from . import renderer
except ImportError:
    # Running uninstalled?
    import renderer
55

56

57
SAMPLE_DURATION = Gst.SECOND / 100
58

59
THUMB_MARGIN_PX = 3
60 61 62
THUMB_HEIGHT = EXPANDED_SIZE - 2 * THUMB_MARGIN_PX
THUMB_PERIOD = int(Gst.SECOND / 2)
assert Gst.SECOND % THUMB_PERIOD == 0
63 64 65
# For the waveforms, ensures we always have a little extra surface when
# scrolling while playing.
MARGIN = 500
66

67 68 69 70 71
PREVIEW_GENERATOR_SIGNALS = {
    "done": (GObject.SIGNAL_RUN_LAST, None, ()),
    "error": (GObject.SIGNAL_RUN_LAST, None, ()),
}

72 73 74 75 76 77 78
GlobalSettings.addConfigSection("previewers")

GlobalSettings.addConfigOption("previewers_max_cpu",
                               section="previewers",
                               key="max-cpu-usage",
                               default=90)

79

80
class PreviewerBin(Gst.Bin, Loggable):
81
    """Baseclass for elements gathering datas to create previews."""
82 83 84 85 86 87 88 89 90
    def __init__(self, bin_desc):
        Gst.Bin.__init__(self)
        Loggable.__init__(self)

        self.internal_bin = Gst.parse_bin_from_description(bin_desc, True)
        self.add(self.internal_bin)
        self.add_pad(Gst.GhostPad.new(None, self.internal_bin.sinkpads[0]))
        self.add_pad(Gst.GhostPad.new(None, self.internal_bin.srcpads[0]))

91
    def finalize(self, proxy=None):
92
        """Finalizes the previewer, saving data to the disk if needed."""
93 94 95 96
        pass


class ThumbnailBin(PreviewerBin):
97 98
    """Bin to generate and save thumbnails to an SQLite database."""

99 100 101 102 103
    __gproperties__ = {
        "uri": (str,
                "uri of the media file",
                "A URI",
                "",
104
                GObject.ParamFlags.READWRITE),
105 106 107 108 109 110 111 112 113 114
    }

    def __init__(self, bin_desc="videoconvert ! videorate ! "
                 "videoscale method=lanczos ! "
                 "capsfilter caps=video/x-raw,format=(string)RGBA,"
                 "height=(int)%d,pixel-aspect-ratio=(fraction)1/1,"
                 "framerate=2/1 ! gdkpixbufsink name=gdkpixbufsink " %
                 THUMB_HEIGHT):
        PreviewerBin.__init__(self, bin_desc)

115
        self.uri = None
116 117 118
        self.thumb_cache = None
        self.gdkpixbufsink = self.internal_bin.get_by_name("gdkpixbufsink")

119
    def __addThumbnail(self, message):
120 121 122 123 124 125 126 127 128 129
        struct = message.get_structure()
        struct_name = struct.get_name()
        if struct_name == "pixbuf":
            stream_time = struct.get_value("stream-time")
            self.log("%s new thumbnail %s", self.uri, stream_time)
            pixbuf = struct.get_value("pixbuf")
            self.thumb_cache[stream_time] = pixbuf

        return False

130
    # pylint: disable=arguments-differ
131 132 133
    def do_post_message(self, message):
        if message.type == Gst.MessageType.ELEMENT and \
                message.src == self.gdkpixbufsink:
134
            GLib.idle_add(self.__addThumbnail, message)
135 136 137

        return Gst.Bin.do_post_message(self, message)

138
    def finalize(self, proxy=None):
139
        """Finalizes the previewer, saving data to file if needed."""
140 141 142 143 144 145 146
        self.thumb_cache.commit()
        if proxy:
            self.thumb_cache.copy(proxy.get_id())

    def do_get_property(self, prop):
        if prop.name == 'uri':
            return self.uri
147 148

        raise AttributeError('unknown property %s' % prop.name)
149 150 151 152

    def do_set_property(self, prop, value):
        if prop.name == 'uri':
            self.uri = value
153
            self.thumb_cache = ThumbnailCache.get(self.uri)
154 155 156 157 158
        else:
            raise AttributeError('unknown property %s' % prop.name)


class TeedThumbnailBin(ThumbnailBin):
159
    """Bin to generate and save thumbnails to an SQLite database."""
160

161 162 163 164 165 166 167 168 169 170 171
    def __init__(self):
        ThumbnailBin.__init__(
            self, bin_desc="tee name=t ! queue  "
            "max-size-buffers=0 max-size-bytes=0 max-size-time=0  ! "
            "videoconvert ! videorate ! videoscale method=lanczos ! "
            "capsfilter caps=video/x-raw,format=(string)RGBA,height=(int)%d,"
            "pixel-aspect-ratio=(fraction)1/1,"
            "framerate=2/1 ! gdkpixbufsink name=gdkpixbufsink "
            "t. ! queue " % THUMB_HEIGHT)


172
# pylint: disable=too-many-instance-attributes
173
class WaveformPreviewer(PreviewerBin):
174
    """Bin to generate and save waveforms as a .npy file."""
175

176 177 178 179 180
    __gproperties__ = {
        "uri": (str,
                "uri of the media file",
                "A URI",
                "",
181
                GObject.ParamFlags.READWRITE),
182 183 184
        "duration": (GObject.TYPE_UINT64,
                     "Duration",
                     "Duration",
185
                     0, GLib.MAXUINT64 - 1, 0, GObject.ParamFlags.READWRITE)
186 187 188 189
    }

    def __init__(self):
        PreviewerBin.__init__(self,
190 191
                              "audioconvert ! audioresample ! "
                              "audio/x-raw,channels=1 ! level name=level"
192 193 194 195 196 197 198 199
                              " ! audioconvert ! audioresample")
        self.level = self.internal_bin.get_by_name("level")
        self.debug("Creating waveforms!!")
        self.peaks = None

        self.uri = None
        self.wavefile = None
        self.passthrough = False
200
        self.samples = []
201 202
        self.n_samples = 0
        self.duration = 0
203
        self.prev_pos = 0
204 205 206 207

    def do_get_property(self, prop):
        if prop.name == 'uri':
            return self.uri
208 209

        if prop.name == 'duration':
210
            return self.duration
211 212

        raise AttributeError('unknown property %s' % prop.name)
213 214 215 216 217 218 219 220

    def do_set_property(self, prop, value):
        if prop.name == 'uri':
            self.uri = value
            self.wavefile = get_wavefile_location_for_uri(self.uri)
            self.passthrough = os.path.exists(self.wavefile)
        elif prop.name == 'duration':
            self.duration = value
221
            self.n_samples = self.duration / SAMPLE_DURATION
222 223 224
        else:
            raise AttributeError('unknown property %s' % prop.name)

225
    # pylint: disable=arguments-differ
226 227 228 229
    def do_post_message(self, message):
        if not self.passthrough and \
                message.type == Gst.MessageType.ELEMENT and \
                message.src == self.level:
230 231 232 233
            struct = message.get_structure()
            peaks = None
            if struct:
                peaks = struct.get_value("rms")
234

235 236
            if peaks:
                stream_time = struct.get_value("stream-time")
237 238 239

                if self.peaks is None:
                    self.peaks = []
240 241
                    for unused_channel in peaks:
                        self.peaks.append([0] * int(self.n_samples))
242

243
                pos = int(stream_time / SAMPLE_DURATION)
244
                if pos >= len(self.peaks[0]):
245
                    return False
246

247
                for i, val in enumerate(peaks):
248 249 250
                    if val < 0:
                        val = 10 ** (val / 20) * 100
                    else:
251 252
                        val = self.peaks[i][pos - 1]

253
                    # Linearly joins values between to known samples values.
254
                    unknowns = range(self.prev_pos + 1, pos)
255
                    if unknowns:
256 257 258 259 260 261 262 263
                        prev_val = self.peaks[i][self.prev_pos]
                        linear_const = (val - prev_val) / len(unknowns)
                        for temppos in unknowns:
                            self.peaks[i][temppos] = self.peaks[i][temppos - 1] + linear_const

                    self.peaks[i][pos] = val

                self.prev_pos = pos
264 265 266 267

        return Gst.Bin.do_post_message(self, message)

    def finalize(self, proxy=None):
268
        """Finalizes the previewer, saving data to file if needed."""
269 270 271
        if not self.passthrough and self.peaks:
            # Let's go mono.
            if len(self.peaks) > 1:
272
                samples = (numpy.array(self.peaks[0]) + numpy.array(self.peaks[1])) / 2
273 274 275 276 277
            else:
                samples = numpy.array(self.peaks[0])

            self.samples = list(samples)
            with open(self.wavefile, 'wb') as wavefile:
278
                numpy.save(wavefile, samples)
279

280
        if proxy and not proxy.get_error():
281 282
            proxy_wavefile = get_wavefile_location_for_uri(proxy.get_id())
            self.debug("symlinking %s and %s", self.wavefile, proxy_wavefile)
283 284 285 286
            try:
                os.remove(proxy_wavefile)
            except FileNotFoundError:
                pass
287 288 289 290 291 292 293 294 295 296 297
            os.symlink(self.wavefile, proxy_wavefile)


Gst.Element.register(None, "waveformbin", Gst.Rank.NONE,
                     WaveformPreviewer)
Gst.Element.register(None, "thumbnailbin", Gst.Rank.NONE,
                     ThumbnailBin)
Gst.Element.register(None, "teedthumbnailbin", Gst.Rank.NONE,
                     TeedThumbnailBin)


298
class PreviewGeneratorManager(Loggable):
299
    """Manager for running the previewers."""
300

301
    def __init__(self):
302 303
        Loggable.__init__(self)

304
        # The current Previewer per GES.TrackType.
305
        self._current_previewers = {}
306
        # The queue of Previewers.
307
        self._previewers = {
308 309 310
            GES.TrackType.AUDIO: [],
            GES.TrackType.VIDEO: []
        }
311
        self._running = True
312

313 314
    def add_previewer(self, previewer):
        """Adds the specified previewer to the queue.
315 316

        Args:
317
            previewer (Previewer): The previewer to control.
318
        """
319
        track_type = previewer.track_type
320

321 322
        current = self._current_previewers.get(track_type)
        if previewer in self._previewers[track_type] or previewer is current:
323
            # Already in the queue or already processing.
324 325
            return

326 327
        if not self._previewers[track_type] and current is None:
            self._start_previewer(previewer)
328
        else:
329
            self._previewers[track_type].insert(0, previewer)
330

331 332 333
    def _start_previewer(self, previewer):
        self._current_previewers[previewer.track_type] = previewer
        previewer.connect("done", self.__previewer_done_cb)
334
        previewer.start_generation()
335

336 337 338 339
    @contextlib.contextmanager
    def paused(self, interrupt=False):
        """Pauses (and flushes if interrupt=True) managed previewers."""
        if interrupt:
340
            for previewer in list(self._current_previewers.values()):
341
                previewer.stop_generation()
342 343 344

            for previewers in self._previewers.values():
                for previewer in previewers:
345
                    previewer.stop_generation()
346 347 348 349 350 351 352
        else:
            for previewer in list(self._current_previewers.values()):
                previewer.pause_generation()

            for previewers in self._previewers.values():
                for previewer in previewers:
                    previewer.pause_generation()
353

354 355 356 357 358 359 360 361 362 363
        try:
            self._running = False
            yield
        except:
            self.warning("An exception occurred while the previewer was paused")
            raise
        finally:
            self._running = True
            for track_type in self._previewers:
                self.__start_next_previewer(track_type)
364

365
    def __previewer_done_cb(self, previewer):
366 367 368
        self.__start_next_previewer(previewer.track_type)

    def __start_next_previewer(self, track_type):
369 370 371
        next_previewer = self._current_previewers.pop(track_type, None)
        if next_previewer:
            next_previewer.disconnect_by_func(self.__previewer_done_cb)
372

373 374 375
        if not self._running:
            return

376 377
        if self._previewers[track_type]:
            self._start_previewer(self._previewers[track_type].pop())
378 379


380
class Previewer(Gtk.Layout):
381
    """Base class for previewers.
382

383 384
    Attributes:
        track_type (GES.TrackType): The type of content.
385 386
    """

387
    # We only need one PreviewGeneratorManager to manage all previewers.
388
    manager = PreviewGeneratorManager()
389

390
    def __init__(self, track_type, max_cpu_usage):
391
        Gtk.Layout.__init__(self)
392

393
        self.track_type = track_type
394
        self._max_cpu_usage = max_cpu_usage
395

396
    def start_generation(self):
397
        """Starts preview generation."""
398
        raise NotImplementedError
399

400
    def stop_generation(self):
401
        """Stops preview generation."""
402
        raise NotImplementedError
403

404
    def become_controlled(self):
405
        """Lets the PreviewGeneratorManager control our execution."""
406
        Previewer.manager.add_previewer(self)
407

408
    def set_selected(self, selected):
409
        """Marks this instance as being selected."""
410 411
        pass

412 413 414 415
    def pause_generation(self):
        """Pauses preview generation."""
        pass

416

417
class VideoPreviewer(Previewer, Zoomable, Loggable):
418 419 420 421
    """A video previewer widget, drawing thumbnails.

    Attributes:
        ges_elem (GES.TrackElement): The previewed element.
422
        thumbs (dict): Maps (quantized) times to Thumbnail widgets.
423
        thumb_cache (ThumbnailCache): The pixmaps persistent cache.
424
    """
425 426

    # We could define them in Previewer, but for some reason they are ignored.
427 428
    __gsignals__ = PREVIEW_GENERATOR_SIGNALS

429 430
    def __init__(self, ges_elem, max_cpu_usage):
        Previewer.__init__(self, GES.TrackType.VIDEO, max_cpu_usage)
431 432
        Zoomable.__init__(self)
        Loggable.__init__(self)
433

434
        self.ges_elem = ges_elem
435

436
        # Guard against malformed URIs
437
        self.uri = quote_uri(get_proxy_target(ges_elem).props.id)
438

439
        self.__start_id = 0
440
        self.__preroll_timeout_id = 0
441
        self._thumb_cb_id = 0
442

443
        # The thumbs to be generated.
444
        self.queue = []
445 446
        # The position for which a thumbnail is currently being generated.
        self.position = -1
447 448
        # The positions for which we failed to get a pixbuf.
        self.failures = set()
449
        self._thumb_cb_id = None
450

451
        self.thumbs = {}
452
        self.thumb_height = THUMB_HEIGHT
453
        self.thumb_width = 0
454

455 456
        self.__image_pixbuf = None
        if not isinstance(ges_elem, GES.ImageSource):
457 458 459 460 461
            self.thumb_cache = ThumbnailCache.get(self.uri)
            self._ensure_proxy_thumbnails_cache()
            self.thumb_width, unused_height = self.thumb_cache.image_size
        self.pipeline = None
        self.gdkpixbufsink = None
462

463
        self.cpu_usage_tracker = CPUUsageTracker()
464 465
        # Initial delay before generating the next thumbnail, in millis.
        self.interval = 500
466

467
        # Connect signals and fire things up
468
        self.ges_elem.connect("notify::in-point", self._inpoint_changed_cb)
469
        self.ges_elem.connect("notify::duration", self._duration_changed_cb)
470

471
        self.become_controlled()
472

473
        self.connect("notify::height-request", self._height_changed_cb)
474

475 476 477 478
    def pause_generation(self):
        if self.pipeline:
            self.pipeline.set_state(Gst.State.READY)

479
    def _setup_pipeline(self):
480
        """Creates the pipeline.
481 482 483 484

        It has the form "playbin ! thumbnailsink" where thumbnailsink
        is a Bin made out of "videorate ! capsfilter ! gdkpixbufsink"
        """
485 486 487 488 489 490
        if self.pipeline:
            # Generation was just PAUSED... keep going
            # bringing the pipeline back to PAUSED.
            self.pipeline.set_state(Gst.State.PAUSED)
            return

491
        pipeline = Gst.parse_launch(
492
            "uridecodebin uri={uri} name=decode ! "
493 494 495 496
            "videoconvert ! "
            "videorate ! "
            "videoscale method=lanczos ! "
            "capsfilter caps=video/x-raw,format=(string)RGBA,height=(int){height},"
497 498 499 500 501
            "pixel-aspect-ratio=(fraction)1/1,framerate={thumbs_per_second}/1 ! "
            "gdkpixbufsink name=gdkpixbufsink".format(
                uri=self.uri,
                height=self.thumb_height,
                thumbs_per_second=int(Gst.SECOND / THUMB_PERIOD)))
502

503 504
        # Get the gdkpixbufsink which contains the the sinkpad.
        self.gdkpixbufsink = pipeline.get_by_name("gdkpixbufsink")
505

506
        decode = pipeline.get_by_name("decode")
507
        decode.connect("autoplug-select", self._autoplug_select_cb)
508

509 510
        self.__preroll_timeout_id = GLib.timeout_add_seconds(MAX_BRINGING_TO_PAUSED_DURATION,
                                                             self.__preroll_timed_out_cb)
511 512 513
        pipeline.get_bus().add_signal_watch()
        pipeline.get_bus().connect("message", self.__bus_message_cb)
        pipeline.set_state(Gst.State.PAUSED)
514
        self.pipeline = pipeline
515

516
    def _schedule_next_thumb_generation(self):
517
        """Schedules the generation of the next thumbnail, or stop.
518 519 520

        Checks the CPU usage and adjusts the waiting time at which the next
        thumbnail will be generated +/- 10%. Even then, it will only
521 522
        happen when the gobject loop is idle to avoid blocking the UI.
        """
523 524 525 526
        if self._thumb_cb_id is not None:
            # A thumb has already been scheduled.
            return

527 528 529 530 531 532
        if not self.queue:
            # Nothing left to do.
            self.debug("Thumbnails generation complete")
            self.stop_generation()
            return

533
        usage_percent = self.cpu_usage_tracker.usage()
534
        if usage_percent < self._max_cpu_usage:
535
            self.interval *= 0.9
536 537
            self.log("Thumbnailing sped up to a %.1f ms interval for `%s`",
                     self.interval, path_from_uri(self.uri))
538 539
        else:
            self.interval *= 1.1
540 541
            self.log("Thumbnailing slowed down to a %.1f ms interval for `%s`",
                     self.interval, path_from_uri(self.uri))
542
        self.cpu_usage_tracker.reset()
543
        self._thumb_cb_id = GLib.timeout_add(self.interval,
544
                                             self._create_next_thumb_cb,
545
                                             priority=GLib.PRIORITY_LOW)
546

547 548
    def _start_thumbnailing_cb(self):
        if not self.__start_id:
549 550 551
            # Can happen if stopGeneration is called because the clip has been
            # removed from the timeline after the PreviewGeneratorManager
            # started this job.
552
            return False
553

554 555 556
        self.__start_id = None

        if isinstance(self.ges_elem, GES.ImageSource):
557
            self.debug("Generating thumbnail for image: %s", path_from_uri(self.uri))
558 559 560
            self.__image_pixbuf = GdkPixbuf.Pixbuf.new_from_file_at_scale(
                Gst.uri_get_location(self.uri), -1, self.thumb_height, True)
            self.thumb_width = self.__image_pixbuf.props.width
561
            self._update_thumbnails()
562
            self.emit("done")
563
        else:
564 565
            if not self.thumb_width:
                self.debug("Finding thumb width")
566 567
                self._setup_pipeline()
                return False
568

569 570
            # Update the thumbnails with what we already have, if anything.
            self._update_thumbnails()
571 572 573 574
            if self.queue:
                self.debug("Generating thumbnails for video: %s, %s", path_from_uri(self.uri), self.queue)
                # When the pipeline status is set to PAUSED,
                # the first thumbnail generation will be scheduled.
575
                self._setup_pipeline()
576 577
            else:
                self.emit("done")
578

579 580 581
        # Stop calling me, I started already.
        return False

582
    def _create_next_thumb_cb(self):
583 584
        """Creates a missing thumbnail."""
        self._thumb_cb_id = None
585

586 587 588 589 590 591 592 593
        try:
            self.position = self.queue.pop(0)
        except IndexError:
            # The queue is empty. Can happen if _update_thumbnails
            # has been called in the meanwhile.
            self.stop_generation()
            return False

594
        self.log("Creating thumb at %s", self.position)
595
        self.pipeline.seek(1.0,
596 597
                           Gst.Format.TIME,
                           Gst.SeekFlags.FLUSH | Gst.SeekFlags.ACCURATE,
598
                           Gst.SeekType.SET, self.position,
599
                           Gst.SeekType.NONE, -1)
600

601 602 603
        # Stop calling me.
        # The seek operation will generate an ASYNC_DONE message on the bus,
        # and then the next thumbnail generation operation will be scheduled.
604 605
        return False

606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621
    @property
    def thumb_interval(self):
        """Gets the interval for which a thumbnail is displayed.

        Returns:
            int: a duration in nanos, multiple of THUMB_PERIOD.
        """
        interval = Zoomable.pixelToNs(self.thumb_width + THUMB_MARGIN_PX)
        # Make sure the thumb interval is a multiple of THUMB_PERIOD.
        quantized = quantize(interval, THUMB_PERIOD)
        # Make sure the quantized thumb interval fits
        # the thumb and the margin.
        if quantized < interval:
            quantized += THUMB_PERIOD
        # Make sure we don't show thumbs more often than THUMB_PERIOD.
        return max(THUMB_PERIOD, quantized)
622

623
    def _update_thumbnails(self):
624
        """Updates the thumbnail widgets for the clip at the current zoom."""
625
        if not self.thumb_width:
626 627
            # The thumb_width will be available when pipeline has been started
            # or the __image_pixbuf is ready.
628
            return
629

630
        thumbs = {}
631
        queue = []
632 633
        interval = self.thumb_interval
        element_left = quantize(self.ges_elem.props.in_point, interval)
634
        element_right = self.ges_elem.props.in_point + self.ges_elem.props.duration
635
        y = (self.props.height_request - self.thumb_height) / 2
636
        for position in range(element_left, element_right, interval):
637 638 639 640 641 642 643 644 645
            x = Zoomable.nsToPixel(position) - self.nsToPixel(self.ges_elem.props.in_point)
            try:
                thumb = self.thumbs.pop(position)
                self.move(thumb, x, y)
            except KeyError:
                thumb = Thumbnail(self.thumb_width, self.thumb_height)
                self.put(thumb, x, y)

            thumbs[position] = thumb
646
            if isinstance(self.ges_elem, GES.ImageSource):
647 648
                thumb.set_from_pixbuf(self.__image_pixbuf)
                thumb.set_visible(True)
649 650
            elif position in self.thumb_cache:
                pixbuf = self.thumb_cache[position]
651 652
                thumb.set_from_pixbuf(pixbuf)
                thumb.set_visible(True)
653
            else:
654 655
                if position not in self.failures and position != self.position:
                    queue.append(position)
656 657 658
        for thumb in self.thumbs.values():
            self.remove(thumb)
        self.thumbs = thumbs
659
        self.queue = queue
660 661
        if queue:
            self.become_controlled()
662

663 664
    def _set_pixbuf(self, pixbuf):
        """Sets the pixbuf for the thumbnail at the expected position."""
665 666 667
        position = self.position
        self.position = -1

668 669 670 671 672 673
        try:
            thumb = self.thumbs[position]
        except KeyError:
            # Can happen because we don't stop the pipeline before
            # updating the thumbnails in _update_thumbnails.
            return
674
        thumb.set_from_pixbuf(pixbuf)
675
        self.thumb_cache[position] = pixbuf
676
        self.queue_draw()
677 678

    def zoomChanged(self):
679
        self._update_thumbnails()
680

681
    def __bus_message_cb(self, unused_bus, message):
682 683
        if message.src == self.pipeline and \
                message.type == Gst.MessageType.STATE_CHANGED:
684
            if message.parse_state_changed()[1] == Gst.State.PAUSED:
685
                # The pipeline is ready to be used.
686 687 688 689 690 691 692 693
                if self.__preroll_timeout_id:
                    GLib.source_remove(self.__preroll_timeout_id)
                    self.__preroll_timeout_id = 0
                    sinkpad = self.gdkpixbufsink.get_static_pad("sink")
                    neg_caps = sinkpad.get_current_caps()[0]
                    self.thumb_width = neg_caps["width"]

                self._update_thumbnails()
694 695 696 697 698 699 700 701
        elif message.src == self.gdkpixbufsink and \
                message.type == Gst.MessageType.ELEMENT and \
                self.__preroll_timeout_id == 0:
            # We got a thumbnail pixbuf.
            struct = message.get_structure()
            struct_name = struct.get_name()
            if struct_name == "preroll-pixbuf":
                pixbuf = struct.get_value("pixbuf")
702
                self._set_pixbuf(pixbuf)
703 704
        elif message.src == self.pipeline and \
                message.type == Gst.MessageType.ASYNC_DONE:
705 706 707 708
            if self.position >= 0:
                self.warning("Thumbnail generation failed at %s", self.position)
                self.failures.add(self.position)
                self.position = -1
709
            self._schedule_next_thumb_generation()
710 711
        return Gst.BusSyncReply.PASS

712
    def __preroll_timed_out_cb(self):
713
        self.stop_generation()
714

715
    # pylint: disable=no-self-use
716
    def _autoplug_select_cb(self, unused_decode, unused_pad, unused_caps, factory):
717 718 719 720 721
        # Don't plug audio decoders / parsers.
        if "Audio" in factory.get_klass():
            return True
        return False

722
    def _height_changed_cb(self, unused_widget, unused_param_spec):
723
        self._update_thumbnails()
724

725
    def _inpoint_changed_cb(self, unused_ges_timeline_element, unused_param_spec):
726
        """Handles the changing of the in-point of the clip."""
727
        self._update_thumbnails()
728

729 730 731 732 733
    def _duration_changed_cb(self, unused_ges_timeline_element, unused_param_spec):
        """Handles the changing of the duration of the clip."""
        self._update_thumbnails()

    def set_selected(self, selected):
734 735 736 737 738 739 740 741
        if selected:
            opacity = 0.5
        else:
            opacity = 1.0

        for thumb in self.get_children():
            thumb.props.opacity = opacity

742
    def start_generation(self):
743
        self.debug("Waiting for UI to become idle for: %s",
744 745 746
                   path_from_uri(self.uri))
        self.__start_id = GLib.idle_add(self._start_thumbnailing_cb,
                                        priority=GLib.PRIORITY_LOW)
747

748
    def _ensure_proxy_thumbnails_cache(self):
749
        """Ensures that both the target asset and the proxy assets have caches."""
750
        uri = quote_uri(self.ges_elem.props.uri)
751 752
        if self.uri != uri:
            self.thumb_cache.copy(uri)
753

754
    def stop_generation(self):
755 756 757 758 759 760 761 762 763 764
        if self.__start_id:
            # Cancel the starting.
            GLib.source_remove(self.__start_id)
            self.__start_id = None

        if self.__preroll_timeout_id:
            # Stop waiting for the pipeline to be ready.
            GLib.source_remove(self.__preroll_timeout_id)
            self.__preroll_timeout_id = None

765
        if self._thumb_cb_id:
766
            # Cancel the thumbnailing.
767 768 769 770
            GLib.source_remove(self._thumb_cb_id)
            self._thumb_cb_id = None

        if self.pipeline:
771
            self.pipeline.get_bus().remove_signal_watch()
772 773 774
            self.pipeline.set_state(Gst.State.NULL)
            self.pipeline.get_state(Gst.CLOCK_TIME_NONE)
            self.pipeline = None
775 776

        self._ensure_proxy_thumbnails_cache()
777
        self.emit("done")
778

779
    def release(self):
780
        """Stops preview generation and cleans the object."""
781
        self.stop_generation()
782 783
        Zoomable.__del__(self)

784 785

class Thumbnail(Gtk.Image):
786 787
    """Simple widget representing a Thumbnail."""

788
    def __init__(self, width, height):
789
        Gtk.Image.__init__(self)
790 791
        self.props.width_request = width
        self.props.height_request = height
792

793

794
class ThumbnailCache(Loggable):
795
    """Cache for the thumbnails of an asset.
796

797
    Uses a separate sqlite3 database for each asset.
798
    """
799

800
    # The cache of caches.
801 802
    caches_by_uri = {}

803
    def __init__(self, uri):
804 805
        Loggable.__init__(self)
        self._filehash = hash_file(Gst.uri_get_location(uri))
806
        thumbs_cache_dir = get_dir(os.path.join(xdg_cache_home(), "thumbs"))
807 808
        self._dbfile = os.path.join(thumbs_cache_dir, self._filehash)
        self._db = sqlite3.connect(self._dbfile)
809 810 811 812 813
        self._cur = self._db.cursor()
        self._cur.execute("CREATE TABLE IF NOT EXISTS Thumbs "
                          "(Time INTEGER NOT NULL PRIMARY KEY, "
                          " Jpeg BLOB NOT NULL)")
        # The cached (width, height) of the images.
814
        self._image_size = (0, 0)
815 816
        # The cached positions available in the database.
        self.positions = self.__existing_positions()
817 818
        # The ID of the autosave event.
        self.__autosave_id = None
819 820 821

    def __existing_positions(self):
        self._cur.execute("SELECT Time FROM Thumbs")
822
        return {row[0] for row in self._cur.fetchall()}
823

824 825 826 827 828 829 830 831 832 833 834 835 836 837 838 839 840 841 842 843 844 845
    @classmethod
    def get(cls, obj):
        """Gets a ThumbnailCache for the specified object.

        Args:
            obj (str or GES.UriClipAsset): The object for which to get a cache,
                it can be a string representing a URI, or a GES.UriClipAsset.

        Returns:
            ThumbnailCache: The cache for the object.
        """
        if isinstance(obj, str):
            uri = obj
        elif isinstance(obj, GES.UriClipAsset):
            uri = get_proxy_target(obj).props.id
        else:
            raise ValueError("Unhandled type: %s" % type(obj))

        if uri not in cls.caches_by_uri:
            cls.caches_by_uri[uri] = ThumbnailCache(uri)
        return cls.caches_by_uri[uri]

846
    def copy(self, uri):
847
        """Copies `self` to the specified `uri`.
848 849 850 851

        Args:
            uri (str): The place where to copy/save the ThumbnailCache
        """
852 853 854 855
        filehash = hash_file(Gst.uri_get_location(uri))
        thumbs_cache_dir = get_dir(os.path.join(xdg_cache_home(), "thumbs"))
        dbfile = os.path.join(thumbs_cache_dir, filehash)

856 857 858 859
        try:
            os.remove(dbfile)
        except FileNotFoundError:
            pass
860 861
        os.symlink(self._dbfile, dbfile)

862 863
    @property
    def image_size(self):
864 865
        """Gets the image size.

866
        Returns:
867
            List[int]: The width and height of the images in the cache.
868
        """
869
        if self._image_size[0] is 0:
870 871 872 873 874 875 876 877
            self._cur.execute("SELECT * FROM Thumbs LIMIT 1")
            row = self._cur.fetchone()
            if row:
                pixbuf = self.__pixbuf_from_row(row)
                self._image_size = (pixbuf.get_width(), pixbuf.get_height())
        return self._image_size

    def get_preview_thumbnail(self):
878
        """Gets a thumbnail contained 'at the middle' of the cache."""
879
        if not self.positions:
880 881
            return None

882 883 884
        middle = int(len(self.positions) / 2)
        position = sorted(list(self.positions))[middle]
        return self[position]
885

886 887 888
    @staticmethod
    def __pixbuf_from_row(row):
        """Returns the GdkPixbuf.Pixbuf from the specified row."""
889 890 891 892 893 894 895
        jpeg = row[1]
        loader = GdkPixbuf.PixbufLoader.new()
        loader.write(jpeg)
        loader.close()
        pixbuf = loader.get_pixbuf()
        return pixbuf

896 897 898
    def __contains__(self, position):
        """Returns whether a row for the specified position exists in the DB."""
        return position in self.positions
899

900 901 902
    def __getitem__(self, position):
        """Gets the GdkPixbuf.Pixbuf for the specified position."""
        self._cur.execute("SELECT * FROM Thumbs WHERE Time = ?", (position,))
903
        row = self._cur.fetchone()
904
        if not row:
905 906
            raise KeyError(position)
        return self.__pixbuf_from_row(row)
907

908 909 910
    def __setitem__(self, position, pixbuf):
        """Sets a GdkPixbuf.Pixbuf for the specified position."""
        success, jpeg = pixbuf.save_to_bufferv(
911
            "jpeg", ["quality", None], ["90"])
912 913 914 915
        if not success:
            self.warning("JPEG compression failed")
            return
        blob = sqlite3.Binary(jpeg)
916
        # Replace if a row with the same time already exists.
917 918 919
        self._cur.execute("DELETE FROM Thumbs WHERE  time=?", (position,))
        self._cur.execute("INSERT INTO Thumbs VALUES (?,?)", (position, blob,))
        self.positions.add(position)
920 921 922 923 924 925 926 927 928 929 930 931 932 933 934 935 936 937 938 939
        self._schedule_commit()

    def _schedule_commit(self):
        """Schedules an autosave at a random later time."""
        if self.__autosave_id is not None:
            # A commit is already scheduled.
            return
        # Save after some time, to avoid saving too often.
        # Randomize to avoid concurrent disk writes.
        random_time = random.randrange(10, 20)
        self.__autosave_id = GLib.timeout_add_seconds(random_time, self._autosave_cb)

    def _autosave_cb(self):
        """Handles the autosave event."""
        try:
            self.commit()
        finally:
            self.__autosave_id = None
        # Stop calling me.
        return False
940 941

    def commit(self):
942
        """Saves the cache on disk (in the database)."""
943
        self._db.commit()
944
        self.log("Saved thumbnail cache file: %s", self._filehash)
945

946

947
def get_wavefile_location_for_uri(uri):
948 949
    """Computes the URI where the wave.npy file should be stored."""
    filename = hash_file(Gst.uri_get_location(uri)) + ".wave.npy"
950 951 952 953 954
    cache_dir = get_dir(os.path.join(xdg_cache_home(), "waves"))

    return os.path.join(cache_dir, filename)


955
class AudioPreviewer(Previewer, Zoomable, Loggable):
956
    """Audio previewer using the results from the "level" GStreamer element."""
957 958 959

    __gsignals__ = PREVIEW_GENERATOR_SIGNALS

960 961
    def __init__(self, ges_elem, max_cpu_usage):
        Previewer.__init__(self, GES.TrackType.AUDIO, max_cpu_usage)
962
        Zoomable.__init__(self)
963
        Loggable.__init__(self)
964

965
        self.pipeline = None
966 967
        self._wavebin = None

968
        self.discovered = False
969
        self.ges_elem = ges_elem
970

971
        asset = self.ges_elem.get_parent().get_asset()
972 973 974
        self.n_samples = asset.get_duration() / SAMPLE_DURATION
        self.samples = None
        self.peaks = None
975 976 977 978
        self._start = 0
        self._end = 0
        self._surface_x = 0

979
        # Guard against malformed URIs
980
        self.wavefile = None
981
        self._uri = quote_uri(get_proxy_target(ges_elem).props.id)
982

983
        self._num_failures = 0
984
        self.adapter = None
985 986
        self.surface = None

987 988
        self._force_redraw = True

989 990
        self.ges_elem.connect("notify::in-point", self._inpoint_changed_cb)
        self.connect("notify::height-request", self._height_changed_cb)
991
        self.become_controlled()
992 993 994

    def _inpoint_changed_cb(self, unused_b_element, unused_value):
        self._force_redraw = True
995

996
    def _height_changed_cb(self, unused_widget, unused_param_spec):
997
        self._force_redraw = True
998

999
    def _startLevelsDiscovery(self):
1000
        filename = get_wavefile_location_for_uri(self._uri)
1001

1002
        if os.path.exists(filename):
1003
            with open(filename, "rb") as samples:
1004
                self.samples = list(numpy.load(samples))
1005 1006 1007
            self._startRendering()
        else:
            self.wavefile = filename
1008
            self._launchPipeline()
1009

1010
    def _launchPipeline(self):
1011
        self.debug(
1012
            'Now generating waveforms for: %s', path_from_uri(self._uri))
1013 1014 1015
        self.pipeline = Gst.parse_launch("uridecodebin name=decode uri=" +
                                         self._uri + " ! waveformbin name=wave"
                                         " ! fakesink qos=false name=faked")
1016 1017 1018 1019
        # This line is necessary so we can instantiate GstTranscoder's
        # GstCpuThrottlingClock below.
        Gst.ElementFactory.make("uritranscodebin", None)
        clock = GObject.new(GObject.type_from_name("GstCpuThrottlingClock"))
1020
        clock.props.cpu_usage = self._max_cpu_usage
1021
        self.pipeline.use_clock(clock)
1022 1023
        faked = self.pipeline.get_by_name("faked")
        faked.props.sync = True
1024
        self._wavebin = self.pipeline.get_by_name("wave")
1025
        asset = self.ges_elem.get_asset().get_filesource_asset()
1026 1027
        self._wavebin.props.uri = asset.get_id()
        self._wavebin.props.duration = asset.get_duration()
1028
        decode = self.pipeline.get_by_name("decode")
1029
        decode.connect("autoplug-select", self._autoplug_select_cb)
1030 1031
        bus = self.pipeline.get_bus()
        bus.add_signal_watch()
1032

1033
        self.n_samples = asset.get_duration() / SAMPLE_DURATION
1034
        bus.connect("message", self._busMessageCb)
1035 1036

    def zoomChanged(self):
1037
        self._force_redraw = True
1038

1039
    def _prepareSamples(self):
1040 1041
        proxy = self.ges_elem.get_parent().get_asset().get_proxy_target()
        self._wavebin.finalize(proxy=proxy)
1042
        self.samples = self._wavebin.samples
1043 1044

    def _startRendering(self):
1045
        self.n_samples = len(self.samples)
1046 1047 1048
        self.discovered = True
        if self.adapter:
            self.adapter.stop()
1049
        self.queue_draw()
1050

1051
    def _busMessageCb(self, bus, message):
1052
        if message.type == Gst.MessageType.EOS:
1053 1054
            self._prepareSamples()
            self._startRendering()
1055
            self.stop_generation()
1056 1057

        elif message.type == Gst.MessageType.ERROR:
1058 1059
            if self.adapter:
                self.adapter.stop()
1060
                self.adapter = None
1061
            # Something went wrong TODO : recover
1062
            self.stop_generation()
1063 1064 1065 1066 1067 1068
            self._num_failures += 1
            if self._num_failures < 2:
                self.warning("Issue during waveforms generation: %s"
                             " for the %ith time, trying again with no rate "
                             " modulation", message.parse_error(),
                             self._num_failures)
1069
                bus.disconnect_by_func(self._busMessageCb)
1070
                self._launchPipeline()
1071
                self.become_controlled()
1072
            else:
1073 1074 1075 1076
                if self.pipeline:
                    Gst.debug_bin_to_dot_file_with_ts(self.pipeline,
                                                      Gst.DebugGraphDetails.ALL,
                                                      "error-generating-waveforms")
1077 1078
                self.error("Aborting due to waveforms generation issue: %s",
                           message.parse_error())
1079

1080
    # pylint: disable=no-self-use
1081
    def _autoplug_select_cb(self, unused_decode, unused_pad, unused_caps, factory):
1082 1083 1084 1085 1086
        # Don't plug video decoders / parsers.
        if "Video" in factory.get_klass():
            return True
        return False

1087
    def _get_num_inpoint_samples(self):
1088 1089 1090
        if self.ges_elem.props.in_point:
            asset_duration = self.ges_elem.get_asset().get_filesource_asset().get_duration()
            return int(self.n_samples / (float(asset_duration) / float(self.ges_elem.props.in_point)))
1091 1092 1093

        return 0

1094
    # pylint: disable=arguments-differ
1095
    def do_draw(self, context):
1096 1097 1098
        if not self.discovered:
            return

1099
        clipped_rect = Gdk.cairo_get_clip_rectangle(context)[1]
1100

1101
        num_inpoint_samples = self._get_num_inpoint_samples()
1102 1103 1104 1105
        drawn_start = self.pixelToNs(clipped_rect.x)
        drawn_duration = self.pixelToNs(clipped_rect.width)
        start = int(drawn_start / SAMPLE_DURATION) + num_inpoint_samples
        end = int((drawn_start + drawn_duration) / SAMPLE_DURATION) + num_inpoint_samples
1106 1107 1108

        if self._force_redraw or self._surface_x > clipped_rect.x or self._end < end:
            self._start = start
1109 1110
            end = int(min(self.n_samples, end + (self.pixelToNs(MARGIN)