dirdiff.py 72.7 KB
Newer Older
1
# Copyright (C) 2002-2006 Stephen Kennedy <stevek@gnome.org>
Kai Willadsen's avatar
Kai Willadsen committed
2
# Copyright (C) 2009-2019 Kai Willadsen <kai.willadsen@gmail.com>
3
#
4
5
6
7
8
9
10
11
12
13
14
15
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 2 of the License, or (at
# your option) any later version.
#
# This program is distributed in the hope that it will be useful, but
# WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
16

17
import collections
18
import copy
19
import errno
20
import functools
21
import logging
steve9000's avatar
steve9000 committed
22
import os
23
import shutil
steve9000's avatar
steve9000 committed
24
import stat
25
import sys
26
import typing
27
import unicodedata
28
29
from collections import namedtuple
from decimal import Decimal
30
from mmap import ACCESS_COPY, mmap
31
from typing import DefaultDict, List, NamedTuple, Optional, Tuple
steve9000's avatar
steve9000 committed
32

33
from gi.repository import Gdk, Gio, GLib, GObject, Gtk
34

35
# TODO: Don't from-import whole modules
36
from meld import misc, tree
37
from meld.conf import _
38
from meld.const import FILE_FILTER_ACTION_FORMAT, MISSING_TIMESTAMP
39
from meld.iohelpers import find_shared_parent_path, trash_or_confirm
40
from meld.melddoc import MeldDoc, open_files_external
41
from meld.misc import all_same, apply_text_filters, with_focused_pane
42
from meld.recent import RecentType
43
from meld.settings import bind_settings, get_meld_settings, settings
44
from meld.treehelpers import refocus_deleted_path, tree_path_as_tuple
45
from meld.ui.cellrenderers import (
46
47
48
    CellRendererByteSize,
    CellRendererDate,
    CellRendererFileMode,
49
    CellRendererISODate,
50
)
51
from meld.ui.emblemcellrenderer import EmblemCellRenderer
52
from meld.ui.util import map_widgets_into_lists
53

54
55
56
if typing.TYPE_CHECKING:
    from meld.ui.pathlabel import PathLabel

57
58
log = logging.getLogger(__name__)

steve9000's avatar
steve9000 committed
59

60
61
62
63
64
65
66
67
class StatItem(namedtuple('StatItem', 'mode size time')):
    __slots__ = ()

    @classmethod
    def _make(cls, stat_result):
        return StatItem(stat.S_IFMT(stat_result.st_mode),
                        stat_result.st_size, stat_result.st_mtime)

68
    def shallow_equal(self, other, time_resolution_ns):
69
70
71
        if self.size != other.size:
            return False

72
73
74
75
76
        # Shortcut to avoid expensive Decimal calculations. 2 seconds is our
        # current accuracy threshold (for VFAT), so should be safe for now.
        if abs(self.time - other.time) > 2:
            return False

77
78
        dectime1 = Decimal(self.time).scaleb(Decimal(9)).quantize(1)
        dectime2 = Decimal(other.time).scaleb(Decimal(9)).quantize(1)
79
80
        mtime1 = dectime1 // time_resolution_ns
        mtime2 = dectime2 // time_resolution_ns
81
82
83

        return mtime1 == mtime2

84
85
86
87

CacheResult = namedtuple('CacheResult', 'stats result')


Stephen Kennedy's avatar
Stephen Kennedy committed
88
_cache = {}
Kai Willadsen's avatar
Kai Willadsen committed
89
90
Same, SameFiltered, DodgySame, DodgyDifferent, Different, FileError = (
    list(range(6)))
91
92
93
# TODO: Get the block size from os.stat
CHUNK_SIZE = 4096

Stephen Kennedy's avatar
Stephen Kennedy committed
94

95
def remove_blank_lines(text):
96
97
98
99
100
    """
    Remove blank lines from text.
    And normalize line ending
    """
    return b'\n'.join(filter(bool, text.splitlines()))
101
102


103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
def _files_contents(files, stats):
    mmaps = []
    is_bin = False
    contents = [b'' for file_obj in files]

    for index, file_and_stat in enumerate(zip(files, stats)):
        file_obj, stat_ = file_and_stat
        # use mmap for files with size > CHUNK_SIZE
        data = b''
        if stat_.size > CHUNK_SIZE:
            data = mmap(file_obj.fileno(), 0, access=ACCESS_COPY)
            mmaps.append(data)
        else:
            data = file_obj.read()
        contents[index] = data

        # Rough test to see whether files are binary.
        chunk_size = min([stat_.size, CHUNK_SIZE])
        if b"\0" in data[:chunk_size]:
            is_bin = True

    return contents, mmaps, is_bin


def _contents_same(contents, file_size):
    other_files_index = list(range(1, len(contents)))
    chunk_range = zip(
        range(0, file_size, CHUNK_SIZE),
Kai Willadsen's avatar
Kai Willadsen committed
131
        range(CHUNK_SIZE, file_size + CHUNK_SIZE, CHUNK_SIZE),
132
133
134
135
136
137
138
139
140
    )

    for start, end in chunk_range:
        chunk = contents[0][start:end]
        for index in other_files_index:
            if not chunk == contents[index][start:end]:
                return Different


Kai Willadsen's avatar
Kai Willadsen committed
141
def _normalize(contents, ignore_blank_lines, regexes=()):
142
143
144
145
146
147
148
    contents = (bytes(c) for c in contents)
    # For probable text files, discard newline differences to match
    if ignore_blank_lines:
        contents = (remove_blank_lines(c) for c in contents)
    else:
        contents = (b"\n".join(c.splitlines()) for c in contents)

149
150
    if regexes:
        contents = (apply_text_filters(c, regexes) for c in contents)
151
152
153
154
        if ignore_blank_lines:
            # We re-remove blank lines here in case applying text
            # filters has caused more lines to be blank.
            contents = (remove_blank_lines(c) for c in contents)
155
156
157
158

    return contents


159
def _files_same(files, regexes, comparison_args):
160
161
162
163
164
165
166
167
    """Determine whether a list of files are the same.

    Possible results are:
      Same: The files are the same
      SameFiltered: The files are identical only after filtering with 'regexes'
      DodgySame: The files are superficially the same (i.e., type, size, mtime)
      DodgyDifferent: The files are superficially different
      FileError: There was a problem reading one or more of the files
Stephen Kennedy's avatar
Stephen Kennedy committed
168
    """
169

170
    if all_same(files):
171
172
173
174
175
        return Same

    files = tuple(files)
    stats = tuple([StatItem._make(os.stat(f)) for f in files])

176
177
178
    shallow_comparison = comparison_args['shallow-comparison']
    time_resolution_ns = comparison_args['time-resolution']
    ignore_blank_lines = comparison_args['ignore_blank_lines']
179
    apply_text_filters = comparison_args['apply-text-filters']
180

181
    need_contents = ignore_blank_lines or apply_text_filters
182

183
184
    regexes = tuple(regexes) if apply_text_filters else ()

185
186
187
188
189
190
191
192
    # If all entries are directories, they are considered to be the same
    if all([stat.S_ISDIR(s.mode) for s in stats]):
        return Same

    # If any entries are not regular files, consider them different
    if not all([stat.S_ISREG(s.mode) for s in stats]):
        return Different

193
    # Compare files superficially if the options tells us to
194
    if shallow_comparison:
195
196
197
198
        all_same_timestamp = all(
            s.shallow_equal(stats[0], time_resolution_ns) for s in stats[1:]
        )
        return DodgySame if all_same_timestamp else Different
199

200
    same_size = all_same([s.size for s in stats])
201
    # If there are no text filters, unequal sizes imply a difference
202
    if not need_contents and not same_size:
203
204
        return Different

205
    # Check the cache before doing the expensive comparison
206
    cache_key = (files, need_contents, regexes, ignore_blank_lines)
207
    cache = _cache.get(cache_key)
208
209
210
211
212
213
    if cache and cache.stats == stats:
        return cache.result

    # Open files and compare bit-by-bit
    result = None

214
    try:
215
216
        mmaps = []
        handles = [open(file_path, "rb") for file_path in files]
217
        try:
218
            contents, mmaps, is_bin = _files_contents(handles, stats)
219

220
221
222
223
224
            # compare files chunk-by-chunk
            if same_size:
                result = _contents_same(contents, stats[0].size)
            else:
                result = Different
225

226
227
228
229
            # normalize and compare files again
            if result == Different and need_contents and not is_bin:
                contents = _normalize(contents, ignore_blank_lines, regexes)
                result = SameFiltered if all_same(contents) else Different
230
231
232
233
234

        # Files are too large; we can't apply filters
        except (MemoryError, OverflowError):
            result = DodgySame if all_same(stats) else DodgyDifferent
        finally:
235
236
            for m in mmaps:
                m.close()
237
238
239
240
241
242
243
244
245
            for h in handles:
                h.close()
    except IOError:
        # Don't cache generic errors as results
        return FileError

    if result is None:
        result = Same

246
    _cache[cache_key] = CacheResult(stats, result)
Stephen Kennedy's avatar
Stephen Kennedy committed
247
248
    return result

249

Kai Willadsen's avatar
Kai Willadsen committed
250
EMBLEM_NEW = "emblem-new"
251
EMBLEM_SELECTED = "emblem-default-symbolic"
252
EMBLEM_SYMLINK = "emblem-symbolic-link"
253

254
255
COL_EMBLEM, COL_EMBLEM_SECONDARY, COL_SIZE, COL_TIME, COL_PERMS, COL_END = (
    range(tree.COL_END, tree.COL_END + 6))
256

steve9000's avatar
steve9000 committed
257

Stephen Kennedy's avatar
Stephen Kennedy committed
258
259
class DirDiffTreeStore(tree.DiffTreeStore):
    def __init__(self, ntree):
260
261
        # FIXME: size should be a GObject.TYPE_UINT64, but we use -1 as a flag
        super().__init__(ntree, [str, str, GObject.TYPE_INT64, float, int])
Stephen Kennedy's avatar
Stephen Kennedy committed
262

263
264
    def add_error(self, parent, msg, pane):
        defaults = {
265
            COL_TIME: MISSING_TIMESTAMP,
266
            COL_SIZE: -1,
Kai Willadsen's avatar
Kai Willadsen committed
267
            COL_PERMS: -1,
268
269
270
        }
        super().add_error(parent, msg, pane, defaults)

271

Dan B's avatar
Dan B committed
272
class ComparisonOptions:
273
274
275
276
277
278
279
280
    def __init__(
        self,
        *,
        ignore_case: bool = False,
        normalize_encoding: bool = False,
    ):
        self.ignore_case = ignore_case
        self.normalize_encoding = normalize_encoding
Dan B's avatar
Dan B committed
281

282

283
class CanonicalListing:
284
285
    """Multi-pane lists with canonicalised matching and error detection"""

286
287
288
289
    items: DefaultDict[str, List[Optional[str]]]
    errors: List[Tuple[int, str, str]]

    def __init__(self, n: int, options: ComparisonOptions):
290
291
        self.items = collections.defaultdict(lambda: [None] * n)
        self.errors = []
292
        self.options = options
293

294
    def add(self, pane: int, item: str):
Dan B's avatar
Dan B committed
295
        # normalize the name depending on settings
296
        ci = item
297
        if self.options.ignore_case:
298
            ci = ci.lower()
299
        if self.options.normalize_encoding:
Dan B's avatar
Dan B committed
300
301
            # NFC or NFD will work here, changing all composed or decomposed
            # characters to the same set for matching only.
302
            ci = unicodedata.normalize('NFC', ci)
303

304
        # add the item to the comparison tree
305
306
        existing_item = self.items[ci][pane]
        if existing_item is None:
307
308
            self.items[ci][pane] = item
        else:
309
            self.errors.append((pane, item, existing_item))
310
311

    def get(self):
312
313
314
315
        def filled(seq):
            fill_value = next(s for s in seq if s)
            return tuple(s or fill_value for s in seq)

316
        return sorted(filled(v) for v in self.items.values())
317
318


319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
class ComparisonMarker(NamedTuple):
    """A stable row + pane marker

    This marker is used for selecting a specific file or folder when
    the user wants to compare paths that don't have matching names, and
    so aren't aligned in our tree view.
    """

    pane: int
    row: Gtk.TreeRowReference

    def get_iter(self) -> Gtk.TreeIter:
        return self.row.get_model().get_iter(self.row.get_path())

    def matches_iter(self, pane: int, it: Gtk.TreeIter) -> bool:
        return (
            pane == self.pane and
            self.row.get_model().get_path(it) == self.row.get_path()
        )

    @classmethod
    def from_selection(
        cls,
        treeview: Gtk.TreeView,
        pane: int,
    ) -> "ComparisonMarker":

        if pane is None or pane == -1:
            raise ValueError("Invalid pane for marker")

        model = treeview.get_model()
        _, selected_paths = treeview.get_selection().get_selected_rows()

        # We'll assume that in any multi-select, the first row was the
        # intended mark.
        selected_row = Gtk.TreeRowReference.new(model, selected_paths[0])

        return cls(
            pane=pane,
            row=selected_row,
        )


362
@Gtk.Template(resource_path='/org/gnome/meld/ui/dirdiff.ui')
363
class DirDiff(Gtk.VBox, tree.TreeviewCommon, MeldDoc):
364
365
366

    __gtype_name__ = "DirDiff"

367
368
369
370
    close_signal = MeldDoc.close_signal
    create_diff_signal = MeldDoc.create_diff_signal
    file_changed_signal = MeldDoc.file_changed_signal
    label_changed = MeldDoc.label_changed
371
    move_diff = MeldDoc.move_diff
372
373
    tab_state_changed = MeldDoc.tab_state_changed

374
375
376
377
378
    __gsettings_bindings__ = (
        ('folder-ignore-symlinks', 'ignore-symlinks'),
        ('folder-shallow-comparison', 'shallow-comparison'),
        ('folder-time-resolution', 'time-resolution'),
        ('folder-status-filters', 'status-filters'),
379
        ('folder-filter-text', 'apply-text-filters'),
380
381
382
        ('ignore-blank-lines', 'ignore-blank-lines'),
    )

383
    apply_text_filters = GObject.Property(
384
385
386
387
388
389
390
        type=bool,
        nick="Apply text filters",
        blurb=(
            "Whether text filters and other text sanitisation preferences "
            "should be applied when comparing file contents"),
        default=False,
    )
391
392
393
394
395
    folders: List[Optional[Gio.File]] = GObject.Property(
        type=object,
        nick="Folders being compared",
        blurb="List of folders being compared, as GFiles",
    )
396
    ignore_blank_lines = GObject.Property(
397
398
399
400
401
        type=bool,
        nick="Ignore blank lines",
        blurb="Whether to ignore blank lines when comparing file contents",
        default=False,
    )
402
    ignore_symlinks = GObject.Property(
403
404
405
406
407
        type=bool,
        nick="Ignore symbolic links",
        blurb="Whether to follow symbolic links when comparing folders",
        default=False,
    )
408
    shallow_comparison = GObject.Property(
409
410
411
412
413
        type=bool,
        nick="Use shallow comparison",
        blurb="Whether to compare files based solely on size and mtime",
        default=False,
    )
414
    status_filters = GObject.Property(
415
416
417
418
        type=GObject.TYPE_STRV,
        nick="File status filters",
        blurb="Files with these statuses will be shown by the comparison.",
    )
419
    time_resolution = GObject.Property(
420
421
422
423
424
425
426
        type=int,
        nick="Time resolution",
        blurb="When comparing based on mtime, the minimum difference in "
              "nanoseconds between two files before they're considered to "
              "have different mtimes.",
        default=100,
    )
steve9000's avatar
steve9000 committed
427

428
429
    show_overview_map = GObject.Property(type=bool, default=True)

430
431
432
    chunkmap0 = Gtk.Template.Child()
    chunkmap1 = Gtk.Template.Child()
    chunkmap2 = Gtk.Template.Child()
433
434
435
436
    folder_label: 'List[PathLabel]'
    folder_label0 = Gtk.Template.Child()
    folder_label1 = Gtk.Template.Child()
    folder_label2 = Gtk.Template.Child()
437
438
439
    folder_open_button0 = Gtk.Template.Child()
    folder_open_button1 = Gtk.Template.Child()
    folder_open_button2 = Gtk.Template.Child()
440
441
442
443
444
445
446
447
448
449
450
451
    treeview0 = Gtk.Template.Child()
    treeview1 = Gtk.Template.Child()
    treeview2 = Gtk.Template.Child()
    scrolledwindow0 = Gtk.Template.Child()
    scrolledwindow1 = Gtk.Template.Child()
    scrolledwindow2 = Gtk.Template.Child()
    linkmap0 = Gtk.Template.Child()
    linkmap1 = Gtk.Template.Child()
    msgarea_mgr0 = Gtk.Template.Child()
    msgarea_mgr1 = Gtk.Template.Child()
    msgarea_mgr2 = Gtk.Template.Child()
    overview_map_revealer = Gtk.Template.Child()
452
453
454
    pane_actionbar0 = Gtk.Template.Child()
    pane_actionbar1 = Gtk.Template.Child()
    pane_actionbar2 = Gtk.Template.Child()
455
456
457
458
459
    vbox0 = Gtk.Template.Child()
    vbox1 = Gtk.Template.Child()
    vbox2 = Gtk.Template.Child()
    dummy_toolbar_linkmap0 = Gtk.Template.Child()
    dummy_toolbar_linkmap1 = Gtk.Template.Child()
460
    toolbar_sourcemap_revealer = Gtk.Template.Child()
461

462
    state_actions = {
463
464
465
466
        tree.STATE_NORMAL: ("normal", "folder-status-same"),
        tree.STATE_NOCHANGE: ("normal", "folder-status-same"),
        tree.STATE_NEW: ("new", "folder-status-new"),
        tree.STATE_MODIFIED: ("modified", "folder-status-modified"),
467
468
    }

469
    def __init__(self, num_panes):
470
471
472
473
474
475
476
477
478
        super().__init__()
        # FIXME:
        # This unimaginable hack exists because GObject (or GTK+?)
        # doesn't actually correctly chain init calls, even if they're
        # not to GObjects. As a workaround, we *should* just be able to
        # put our class first, but because of Gtk.Template we can't do
        # that if it's a GObject, because GObject doesn't support
        # multiple inheritance and we need to inherit from our Widget
        # parent to make Template work.
479
        MeldDoc.__init__(self)
480
        bind_settings(self)
481

482
483
484
485
486
487
488
489
490
        self.view_action_group = Gio.SimpleActionGroup()

        property_actions = (
            ('show-overview-map', self, 'show-overview-map'),
        )
        for action_name, obj, prop_name in property_actions:
            action = Gio.PropertyAction.new(action_name, obj, prop_name)
            self.view_action_group.add_action(action)

491
492
        # Manually handle GAction additions
        actions = (
493
            ('find', self.action_find),
494
            ('folder-collapse', self.action_folder_collapse),
495
            ('folder-compare', self.action_diff),
496
497
            ('folder-mark', self.action_mark),
            ('folder-compare-marked', self.action_diff_marked),
498
499
            ('folder-copy-left', self.action_copy_left),
            ('folder-copy-right', self.action_copy_right),
500
            ('swap-2-panes', self.action_swap),
501
            ('folder-delete', self.action_delete),
502
            ('folder-expand', self.action_folder_expand),
503
            ('next-change', self.action_next_change),
504
            ('next-pane', self.action_next_pane),
505
            ('open-external', self.action_open_external),
506
            ('previous-change', self.action_previous_change),
507
            ('previous-pane', self.action_prev_pane),
Kai Willadsen's avatar
Kai Willadsen committed
508
            ('refresh', self.action_refresh),
509
            ('copy-file-paths', self.action_copy_file_paths),
510
511
512
513
514
515
        )
        for name, callback in actions:
            action = Gio.SimpleAction.new(name, None)
            action.connect('activate', callback)
            self.view_action_group.add_action(action)

516
517
518
519
520
521
522
523
524
        actions = (
            ("folder-status-same", self.action_filter_state_change,
                GLib.Variant.new_boolean(False)),
            ("folder-status-new", self.action_filter_state_change,
                GLib.Variant.new_boolean(False)),
            ("folder-status-modified", self.action_filter_state_change,
                GLib.Variant.new_boolean(False)),
            ("folder-ignore-case", self.action_ignore_case_change,
                GLib.Variant.new_boolean(False)),
525
526
            ("folder-normalize-encoding", self.action_ignore_case_change,
                GLib.Variant.new_boolean(False)),
527
528
529
530
531
532
        )
        for (name, callback, state) in actions:
            action = Gio.SimpleAction.new_stateful(name, None, state)
            action.connect('change-state', callback)
            self.view_action_group.add_action(action)

533
534
535
536
537
538
        builder = Gtk.Builder.new_from_resource(
            '/org/gnome/meld/ui/dirdiff-menus.ui')
        context_menu = builder.get_object('dirdiff-context-menu')
        self.popup_menu = Gtk.Menu.new_from_model(context_menu)
        self.popup_menu.attach_to_widget(self)

539
540
541
542
        builder = Gtk.Builder.new_from_resource(
            '/org/gnome/meld/ui/dirdiff-actions.ui')
        self.toolbar_actions = builder.get_object('view-toolbar')

543
544
        self.folders = [None, None, None]

545
        self.name_filters = []
546
        self.text_filters = []
547
        self.create_name_filters()
548
        self.create_text_filters()
549
        meld_settings = get_meld_settings()
550
        self.settings_handlers = [
551
552
553
554
            meld_settings.connect(
                "file-filters-changed", self.on_file_filters_changed),
            meld_settings.connect(
                "text-filters-changed", self.on_text_filters_changed)
555
        ]
556

557
558
559
560
561
562
        # Handle overview map visibility binding. Because of how we use
        # grid packing, we need two revealers here instead of the more
        # obvious one.
        revealers = (
            self.toolbar_sourcemap_revealer,
            self.overview_map_revealer,
563
        )
564
565
566
        for revealer in revealers:
            self.bind_property(
                'show-overview-map', revealer, 'reveal-child',
Kai Willadsen's avatar
Kai Willadsen committed
567
568
569
570
                (
                    GObject.BindingFlags.DEFAULT |
                    GObject.BindingFlags.SYNC_CREATE
                ),
571
            )
572

573
574
575
        map_widgets_into_lists(
            self,
            [
576
                "treeview", "folder_label", "scrolledwindow", "chunkmap",
577
                "linkmap", "msgarea_mgr", "vbox", "dummy_toolbar_linkmap",
578
                "pane_actionbar", "folder_open_button",
579
            ],
580
        )
581

582
        self.ensure_style()
583

584
        self.custom_labels = []
steve9000's avatar
steve9000 committed
585
        self.set_num_panes(num_panes)
586

587
588
        self.connect("style-updated", self.model.on_style_updated)
        self.model.on_style_updated(self)
589

590
        self.do_to_others_lock = False
591
        for treeview in self.treeview:
592
            treeview.set_search_equal_func(tree.treeview_search_cb, None)
593
        self.force_cursor_recalculate = False
594
        self.current_path, self.prev_path, self.next_path = None, None, None
595
        self.focus_pane = None
596
        self.row_expansions = set()
Stephen Kennedy's avatar
Stephen Kennedy committed
597

598
599
600
601
602
        # One column-dict for each treeview, for changing visibility and order
        self.columns_dict = [{}, {}, {}]
        for i in range(3):
            col_index = self.model.column_index
            # Create icon and filename CellRenderer
603
            column = Gtk.TreeViewColumn(_("Name"))
604
            column.set_resizable(True)
605
            rentext = Gtk.CellRendererText()
606
            renicon = EmblemCellRenderer()
607
608
            column.pack_start(renicon, False)
            column.pack_start(rentext, True)
609
            column.set_attributes(rentext, markup=col_index(tree.COL_TEXT, i),
610
                                  foreground_rgba=col_index(tree.COL_FG, i),
611
612
613
                                  style=col_index(tree.COL_STYLE, i),
                                  weight=col_index(tree.COL_WEIGHT, i),
                                  strikethrough=col_index(tree.COL_STRIKE, i))
614
615
616
617
618
619
620
            column.set_attributes(
                renicon,
                icon_name=col_index(tree.COL_ICON, i),
                emblem_name=col_index(COL_EMBLEM, i),
                secondary_emblem_name=col_index(COL_EMBLEM_SECONDARY, i),
                icon_tint=col_index(tree.COL_TINT, i)
            )
621
622
623
            self.treeview[i].append_column(column)
            self.columns_dict[i]["name"] = column
            # Create file size CellRenderer
624
            column = Gtk.TreeViewColumn(_("Size"))
625
            column.set_resizable(True)
626
            rentext = CellRendererByteSize()
627
            column.pack_start(rentext, True)
628
            column.set_attributes(rentext, bytesize=col_index(COL_SIZE, i))
629
630
631
            self.treeview[i].append_column(column)
            self.columns_dict[i]["size"] = column
            # Create date-time CellRenderer
632
            column = Gtk.TreeViewColumn(_("Modification time"))
633
            column.set_resizable(True)
634
            rentext = CellRendererDate()
635
            column.pack_start(rentext, True)
636
            column.set_attributes(rentext, timestamp=col_index(COL_TIME, i))
637
638
            self.treeview[i].append_column(column)
            self.columns_dict[i]["modification time"] = column
639
640
641
642
643
644
645
646
            # Create ISO-format date-time CellRenderer
            column = Gtk.TreeViewColumn(_("Modification time (ISO)"))
            column.set_resizable(True)
            rentext = CellRendererISODate()
            column.pack_start(rentext, True)
            column.set_attributes(rentext, timestamp=col_index(COL_TIME, i))
            self.treeview[i].append_column(column)
            self.columns_dict[i]["iso-time"] = column
647
            # Create permissions CellRenderer
648
            column = Gtk.TreeViewColumn(_("Permissions"))
649
            column.set_resizable(True)
650
            rentext = CellRendererFileMode()
651
            column.pack_start(rentext, False)
652
            column.set_attributes(rentext, file_mode=col_index(COL_PERMS, i))
653
654
            self.treeview[i].append_column(column)
            self.columns_dict[i]["permissions"] = column
655

steve9000's avatar
steve9000 committed
656
        for i in range(3):
657
            selection = self.treeview[i].get_selection()
658
            selection.set_mode(Gtk.SelectionMode.MULTIPLE)
659
660
661
662
663
            selection.connect('changed', self.on_treeview_selection_changed, i)
            self.scrolledwindow[i].get_vadjustment().connect(
                "value-changed", self._sync_vscroll)
            self.scrolledwindow[i].get_hadjustment().connect(
                "value-changed", self._sync_hscroll)
Stephen Kennedy's avatar
Stephen Kennedy committed
664
        self.linediffs = [[], []]
665

666
667
668
669
        self.update_treeview_columns(settings, 'folder-columns')
        settings.connect('changed::folder-columns',
                         self.update_treeview_columns)

670
671
672
        self.update_comparator()
        self.connect("notify::shallow-comparison", self.update_comparator)
        self.connect("notify::time-resolution", self.update_comparator)
673
        self.connect("notify::ignore-blank-lines", self.update_comparator)
674
        self.connect("notify::apply-text-filters", self.update_comparator)
675

676
677
        # The list copying and state_filters reset here is because the action
        # toggled callback modifies the state while we're constructing it.
678
        self.state_filters = []
679
        state_filters = []
680
681
682
683
684
685
        for s in self.state_actions:
            if self.state_actions[s][0] in self.props.status_filters:
                state_filters.append(s)
                action_name = self.state_actions[s][1]
                self.set_action_state(
                    action_name, GLib.Variant.new_boolean(True))
686
        self.state_filters = state_filters
687

688
689
        self._scan_in_progress = 0

690
691
        self.marked = None

692
693
694
695
    def queue_draw(self):
        for treeview in self.treeview:
            treeview.queue_draw()

696
697
698
699
    def update_comparator(self, *args):
        comparison_args = {
            'shallow-comparison': self.props.shallow_comparison,
            'time-resolution': self.props.time_resolution,
700
            'apply-text-filters': self.props.apply_text_filters,
701
            'ignore_blank_lines': self.props.ignore_blank_lines,
702
703
704
705
        }
        self.file_compare = functools.partial(
            _files_same, comparison_args=comparison_args)
        self.refresh()
706

707
708
709
    def update_treeview_columns(
        self, settings: Gio.Settings, key: str,
    ) -> None:
710
        """Update the visibility and order of columns"""
711

712
        columns = settings.get_value(key)
713
714
        have_extra_columns = any(visible for name, visible in columns)

715
716
717
718
719
720
721
722
        # Check for columns missing from the settings, special-casing
        # the always-present name column
        configured_columns = [name for name, visible in columns] + ["name"]
        missing_columns = [
            c for c in self.columns_dict[0].keys()
            if c not in configured_columns
        ]

723
724
725
        for i, treeview in enumerate(self.treeview):
            last_column = treeview.get_column(0)
            for column_name, visible in columns:
726
727
728
729
730
                try:
                    current_column = self.columns_dict[i][column_name]
                except KeyError:
                    log.warning(f"Invalid column {column_name} in settings")
                    continue
731
                current_column.set_visible(visible)
732
                treeview.move_column_after(current_column, last_column)
733
                last_column = current_column
734

735
736
737
            for column_name in missing_columns:
                self.columns_dict[i][column_name].set_visible(False)

738
            treeview.set_headers_visible(have_extra_columns)
739

740
741
742
743
    def get_filter_visibility(self) -> Tuple[bool, bool, bool]:
        # TODO: Make text filters available in folder comparison
        return False, True, False

744
    def on_file_filters_changed(self, app):
745
746
747
        relevant_change = self.create_name_filters()
        if relevant_change:
            self.refresh()
748

Stephen Kennedy's avatar
Stephen Kennedy committed
749
    def create_name_filters(self):
750
751
        meld_settings = get_meld_settings()

752
        # Ordering of name filters is irrelevant
753
754
        old_active = set([f.filter_string for f in self.name_filters
                          if f.active])
755
        new_active = set([f.filter_string for f in meld_settings.file_filters
756
                          if f.active])
757
758
        active_filters_changed = old_active != new_active

759
760
761
        # TODO: Rework name_filters to use a map-like structure so that we
        # don't need _action_name_filter_map.
        self._action_name_filter_map = {}
762
        self.name_filters = [copy.copy(f) for f in meld_settings.file_filters]
763
764
765
766
767
768
769
770
771
772
        for i, filt in enumerate(self.name_filters):
            action = Gio.SimpleAction.new_stateful(
                name=FILE_FILTER_ACTION_FORMAT.format(i),
                parameter_type=None,
                state=GLib.Variant.new_boolean(filt.active),
            )
            action.connect('change-state', self._update_name_filter)
            action.set_enabled(filt.filter is not None)
            self.view_action_group.add_action(action)
            self._action_name_filter_map[action] = filt
Stephen Kennedy's avatar
Stephen Kennedy committed
773

774
775
        return active_filters_changed

776
777
778
779
780
781
    def on_text_filters_changed(self, app):
        relevant_change = self.create_text_filters()
        if relevant_change:
            self.refresh()

    def create_text_filters(self):
782
783
        meld_settings = get_meld_settings()

784
785
        # In contrast to file filters, ordering of text filters can matter
        old_active = [f.filter_string for f in self.text_filters if f.active]
786
        new_active = [f.filter_string for f in meld_settings.text_filters
787
                      if f.active]
788
789
        active_filters_changed = old_active != new_active

790
        self.text_filters = [copy.copy(f) for f in meld_settings.text_filters]
791
792
793

        return active_filters_changed

Stephen Kennedy's avatar
Stephen Kennedy committed
794
    def _do_to_others(self, master, objects, methodname, args):
795
796
797
798
799
800
801
802
803
804
805
        if self.do_to_others_lock:
            return

        self.do_to_others_lock = True
        try:
            others = [o for o in objects[:self.num_panes] if o != master]
            for o in others:
                method = getattr(o, methodname)
                method(*args)
        finally:
            self.do_to_others_lock = False
Stephen Kennedy's avatar
Stephen Kennedy committed
806
807

    def _sync_vscroll(self, adjustment):
808
        adjs = [sw.get_vadjustment() for sw in self.scrolledwindow]
809
810
        self._do_to_others(
            adjustment, adjs, "set_value", (int(adjustment.get_value()),))
Stephen Kennedy's avatar
Stephen Kennedy committed
811
812

    def _sync_hscroll(self, adjustment):
813
        adjs = [sw.get_hadjustment() for sw in self.scrolledwindow]
814
815
        self._do_to_others(
            adjustment, adjs, "set_value", (int(adjustment.get_value()),))
Stephen Kennedy's avatar
Stephen Kennedy committed
816
817

    def _get_focused_pane(self):
818
819
820
821
        for i, treeview in enumerate(self.treeview):
            if treeview.is_focus():
                return i
        return None
Stephen Kennedy's avatar
Stephen Kennedy committed
822
823
824

    def file_deleted(self, path, pane):
        # is file still extant in other pane?
Stephen Kennedy's avatar
Stephen Kennedy committed
825
826
        it = self.model.get_iter(path)
        files = self.model.value_paths(it)
Kai Willadsen's avatar
Kai Willadsen committed
827
        is_present = [os.path.exists(f) for f in files]
Stephen Kennedy's avatar
Stephen Kennedy committed
828
        if 1 in is_present:
Stephen Kennedy's avatar
Stephen Kennedy committed
829
            self._update_item_state(it)
Kai Willadsen's avatar
Kai Willadsen committed
830
        else:  # nope its gone
Stephen Kennedy's avatar
Stephen Kennedy committed
831
            self.model.remove(it)
Stephen Kennedy's avatar
Stephen Kennedy committed
832
833

    def file_created(self, path, pane):
Stephen Kennedy's avatar
Stephen Kennedy committed
834
        it = self.model.get_iter(path)
835
836
        root = Gtk.TreePath.new_first()
        while it and self.model.get_path(it) != root:
Kai Willadsen's avatar
Kai Willadsen committed
837
            self._update_item_state(it)
Stephen Kennedy's avatar
Stephen Kennedy committed
838
            it = self.model.iter_parent(it)
Stephen Kennedy's avatar
Stephen Kennedy committed
839

840
    @Gtk.Template.Callback()
841
842
843
844
845
846
847
848
849
    def on_file_selected(
            self, button: Gtk.Button, pane: int, file: Gio.File) -> None:
        self.folders[pane] = file
        self.set_locations()

    def set_locations(self) -> None:
        locations = [f.get_path() for f in self.folders if f]
        if not locations:
            return
steve9000's avatar
steve9000 committed
850
851

        self.set_num_panes(len(locations))
852
853
854
855
856

        parent_path = find_shared_parent_path(self.folders)
        for pane, folder in enumerate(self.folders):
            self.folder_label[pane].set_file(folder)
            self.folder_label[pane].set_parent_file(parent_path)
857
            self.folder_open_button[pane].props.file = folder
858

859
860
861
        # This is difficult to trigger, and to test. Most of the time here we
        # will actually have had UTF-8 from GTK, which has been unicode-ed by
        # the time we get this far. This is a fallback, and may be wrong!
862
        locations = list(locations)
863
        for i, l in enumerate(locations):
864
            if l and not isinstance(l, str):
865
                locations[i] = l.decode(sys.getfilesystemencoding())
866
        locations = [os.path.abspath(l) if l else '' for l in locations]
867

868
        self.current_path = None
869
        self.marked = None
Stephen Kennedy's avatar
Stephen Kennedy committed
870
        self.model.clear()
871
872
        for m in self.msgarea_mgr:
            m.clear()
Stephen Kennedy's avatar
Stephen Kennedy committed
873
        child = self.model.add_entries(None, locations)
874
        self.treeview0.grab_focus()
Stephen Kennedy's avatar
Stephen Kennedy committed
875
        self._update_item_state(child)
876
877
        self.recompute_label()
        self.scheduler.remove_all_tasks()
878
        self._scan_in_progress = 0
879
        self.recursively_update(Gtk.TreePath.new_first())
880

881
    def get_comparison(self):
882
        root = self.model.get_iter_first()
883
        if root:
884
            uris = [Gio.File.new_for_path(d)
885
                    for d in self.model.value_paths(root)]
886
        else:
887
            uris = []
888
        return RecentType.Folder, uris
889

890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
    def mark_in_progress_row(self, it: Gtk.TreeIter) -> None:
        """Mark a tree row as having a scan in progress

        After the scan is finished, `_update_item_state()` must be
        called on the row to restore its actual state.
        """

        for pane in range(self.model.ntree):
            path = self.model.get_value(
                it, self.model.column_index(tree.COL_PATH, pane))
            filename = GLib.markup_escape_text(os.path.basename(path))
            label = _(f"{filename} (scanning…)")

            self.model.set_state(it, pane, tree.STATE_SPINNER, label, True)
            self.model.unsafe_set(it, pane, {
                COL_EMBLEM: None,
                COL_EMBLEM_SECONDARY: None,
                COL_TIME: MISSING_TIMESTAMP,
                COL_SIZE: -1,
                COL_PERMS: -1
            })

Kai Willadsen's avatar
Kai Willadsen committed
912
    def recursively_update(self, path):
Stephen Kennedy's avatar
Stephen Kennedy committed
913
914
        """Recursively update from tree path 'path'.
        """
Kai Willadsen's avatar
Kai Willadsen committed
915
916
        it = self.model.get_iter(path)
        child = self.model.iter_children(it)
Stephen Kennedy's avatar
Stephen Kennedy committed
917
918
        while child:
            self.model.remove(child)
Kai Willadsen's avatar
Kai Willadsen committed
919
            child = self.model.iter_children(it)
920
921
922
923
924
        if self._scan_in_progress == 0:
            # Starting a scan, so set up progress indicator
            self.mark_in_progress_row(it)
        else:
            self._update_item_state(it)
925
        self._scan_in_progress += 1
926
        self.scheduler.add_task(self._search_recursively_iter(path))
steve9000's avatar
steve9000 committed
927

Stephen Kennedy's avatar
Stephen Kennedy committed
928
    def _search_recursively_iter(self, rootpath):
929
930
931
932
        for t in self.treeview:
            sel = t.get_selection()
            sel.unselect_all()

933
934
        yield _('[{label}] Scanning {folder}').format(
            label=self.label_text, folder='')
935
936
        prefixlen = 1 + len(
            self.model.value_path(self.model.get_iter(rootpath), 0))
Kai Willadsen's avatar
Kai Willadsen committed
937
        symlinks_followed = set()
938
939
940
941
        # TODO: This is horrible.
        if isinstance(rootpath, tuple):
            rootpath = Gtk.TreePath(rootpath)
        todo = [rootpath]
942
        expanded = set()
943
944

        shadowed_entries = []
945
        invalid_filenames = []
946

947
948
949
950
951
952
        # TODO: Map these action states to GObject props instead?
        comparison_options = ComparisonOptions(
            ignore_case=self.get_action_state('folder-ignore-case'),
            normalize_encoding=self.get_action_state(
                'folder-normalize-encoding'),
        )
953

Stephen Kennedy's avatar
Stephen Kennedy committed
954
        while len(todo):
Kai Willadsen's avatar
Kai Willadsen committed
955
            todo.sort()  # depth first
956
            path = todo.pop(0)