gitlab-operations.py 7.89 KB
Newer Older
Andrea Veri's avatar
Andrea Veri committed
1
#!/usr/bin/python3
2

3
4
import sys
import gitlab
Bartłomiej Piotrowski's avatar
Bartłomiej Piotrowski committed
5
import re
Andrea Veri's avatar
Andrea Veri committed
6
import os
7
8

sys.path.append('/home/admin/bin')
Andrea Veri's avatar
Andrea Veri committed
9
sys.path.append('/home/admin/bin/git')
Andrea Veri's avatar
Andrea Veri committed
10
import gnome_ldap_utils as Glu
11
12
13
import semi_rdf

from xml.sax import SAXParseException
14
from urllib.error import HTTPError
15

Andrea Veri's avatar
Andrea Veri committed
16
LDAP_GROUP_BASE, LDAP_HOST, LDAP_USER_BASE, LDAP_USER, LDAP_PASSWORD, GITLAB_PRIVATE_RW_TOKEN = os.getenv('LDAP_GROUP_BASE'), \
Andrea Veri's avatar
Andrea Veri committed
17
os.getenv('LDAP_HOST'), os.getenv('LDAP_USER_BASE'), os.getenv('LDAP_USER'), os.getenv('LDAP_PASSWORD'), \
Andrea Veri's avatar
Andrea Veri committed
18
os.getenv('GITLAB_PRIVATE_RW_TOKEN')
19

Andrea Veri's avatar
Andrea Veri committed
20
glu = Glu.Gnome_ldap_utils(LDAP_GROUP_BASE, LDAP_HOST, LDAP_USER_BASE, LDAP_USER, LDAP_PASSWORD)
21
gl = gitlab.Gitlab('https://gitlab.gnome.org', GITLAB_PRIVATE_RW_TOKEN, api_version=4)
22

23
24
25
DOAP = "http://usefulinc.com/ns/doap#"
GNOME = "http://api.gnome.org/doap-extensions#"

Andrea Veri's avatar
Andrea Veri committed
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
committers = glu.get_uids_from_group('gnomecvs')
committers_exceptions = glu.get_uids_from_group('gnomecvs_exceptions')

_committers = dict()
for uid in committers:
    try:
        _committers[uid] = gl.users.list(extern_uid=(f"uid={ uid },{ LDAP_USER_BASE }"), \
            provider='ldapmain')[0].id
    except IndexError:
        continue

_committers_exceptions = dict()
for uid in committers_exceptions:
    try:
        _committers_exceptions[uid] = gl.users.list(extern_uid=(f"uid={ uid },{ LDAP_USER_BASE }"), \
            provider='ldapmain')[0].id
    except IndexError:
        continue

gnome_group = gl.groups.get(8, with_projects=False)
46
47
gnome_group_users = { i.username : i.id for i in gnome_group.members.list(get_all=True) }
gnome_projects = gnome_group.projects.list(get_all=True)
Andrea Veri's avatar
Andrea Veri committed
48

49
l10n_group = gl.groups.get(13336, with_projects=False)
Andrea Veri's avatar
Andrea Veri committed
50
51

for username, id in _committers.items():
52
    if id not in gnome_group_users.values():
Andrea Veri's avatar
Andrea Veri committed
53
        gnome_group.members.create({'user_id': id, 'access_level': gitlab.const.AccessLevel.DEVELOPER})
Andrea Veri's avatar
Andrea Veri committed
54
55
        print(f"Account with username { username } and with id { id } has been added to the GNOME group")

56
        try:
Andrea Veri's avatar
Andrea Veri committed
57
            l10n_group.members.create({'user_id': id, 'access_level': gitlab.const.AccessLevel.REPORTER})
58
59
60
61
            print(f"Account with username { username } and id { id } has been added to the Teams/Translation group")
        except gitlab.exceptions.GitlabCreateError as e:
            if e.response_code == 409:
                pass
Andrea Veri's avatar
Andrea Veri committed
62

63
for username, id in gnome_group_users.items():
Andrea Veri's avatar
Andrea Veri committed
64
65
66
67
68
    if id not in _committers.values() and id not in _committers_exceptions.values():
        role = gnome_group.members.get(id).access_level
        if role != 20 and id != 1:
            gnome_group.members.delete(id)
            print(f"Account with username { username } with id { id } has been removed from the GNOME group")
69

70
        try:
71
            l10n_group.members.delete(id)
Andrea Veri's avatar
Andrea Veri committed
72
            print(f"Account with username { username } and id { id } has been removed from the Teams/Translation group")
73
74
75
        except gitlab.exceptions.GitlabDeleteError as e:
            if e.response_code == 404:
                pass
76

77
maints = dict()
Andrea Veri's avatar
Andrea Veri committed
78
for project in gnome_projects:
79
    project_name = project.attributes['path']
80
81
    uids = []

82
    doap_url = 'https://gitlab.gnome.org/GNOME/%s/raw/%s/%s.doap'
83
84
85
86
    default_branch = project.attributes.get('default_branch')
    if not default_branch:
        default_branch = 'master'

87
    # Ensure stable branches are protected
88
    try:
89
        proj = gl.projects.get(project.id)
Andrea Veri's avatar
Andrea Veri committed
90
        branches = [branch.name for branch in proj.branches.list(get_all=True)]
91
        stable_branches_regex = re.compile(r'(gnome-\d-\d\d|gnome-\d\d)')
92
        stable_branches = set(filter(stable_branches_regex.match, branches))
93
        protected_branches = [branch.name for branch in proj.protectedbranches.list(get_all=True)]
94

95
        if default_branch not in protected_branches:
Andrea Veri's avatar
Andrea Veri committed
96
            proj.protectedbranches.create({
97
                'name': default_branch,
98
99
                'merge_access_level': gitlab.const.AccessLevel.DEVELOPER,
                'push_access_level': gitlab.const.AccessLevel.DEVELOPER
Andrea Veri's avatar
Andrea Veri committed
100
            })
101

102
        if len(stable_branches) > 0 and 'gnome-*' not in protected_branches:
Andrea Veri's avatar
Andrea Veri committed
103
            proj.protectedbranches.create({
104
                'name': 'gnome-*',
105
106
                'merge_access_level': gitlab.const.AccessLevel.DEVELOPER,
                'push_access_level': gitlab.const.AccessLevel.DEVELOPER
Andrea Veri's avatar
Andrea Veri committed
107
            })
Bartłomiej Piotrowski's avatar
Bartłomiej Piotrowski committed
108
    except gitlab.exceptions.GitlabListError as e:
109
        # Some projects are not used for git hosting
Bartłomiej Piotrowski's avatar
Bartłomiej Piotrowski committed
110
111
        if e.response_code == 404:
            pass
112

113
114
115
116
    # Ensure container registry is enabled for GNOME projects
    proj.container_registry_enabled = True
    proj.save()

117
    try:
118
        nodes = semi_rdf.read_rdf(doap_url % (project_name, default_branch, project_name))
119
    except (SAXParseException, HTTPError):
120
        nodes = ''
121
122
123
124
125

    for node in nodes:
      if node.name != (DOAP, "Project"):
        continue

126
127
128
129
      for role in [u'maintainer', u'helper']:
          for maint in node.find_properties((DOAP, role)):
              if not isinstance(maint, semi_rdf.Node):
                continue
130

131
132
133
              uid = maint.find_property((GNOME, u'userid'))
              if not isinstance(uid, str):
                continue
134

135
136
              uid = str(uid)
              uids.append(uid)
137

138
              maints[project_name] = uids
139
140

for project in maints:
141
142
143
144
145
146
    try:
        proj = gl.projects.get('GNOME/%s' % project)
    except gitlab.exceptions.GitlabGetError as e:
        if e.response_code == 404:
            continue

147
    for user in maints[project]:
Andrea Veri's avatar
Andrea Veri committed
148
149
        if user in _committers.keys():
            userid = _committers[user]
150
151

            try:
Andrea Veri's avatar
Andrea Veri committed
152
                proj.members.create({'user_id': userid, 'access_level': gitlab.const.AccessLevel.MAINTAINER})
153

Andrea Veri's avatar
Andrea Veri committed
154
                print(f"Landed master level access to { user } against repository { project }")
155
156
157
158
            except gitlab.exceptions.GitlabCreateError as e:
                if e.response_code == 409:
                    if proj.members.get(userid).access_level != 40:
                        proj.members.delete(userid)
Andrea Veri's avatar
Andrea Veri committed
159
                        proj.members.create({'user_id': userid, 'access_level': gitlab.const.AccessLevel.MAINTAINER})
160
161

                        print(f"Landed master level access to { user } against repository { project }")
162
163
164
165
166

    members = proj.members.list()
    members_dict = {}

    for member in members:
167
        identity_found = False
168
        user = gl.users.get(member.attributes['id'])
169
170
171
172

        if len(user.attributes['identities']) > 0:
            for index, _ in enumerate(user.attributes['identities']):
                provider = user.attributes['identities'][index]['provider']
173
                if provider == 'ldapmain':
174
175
176
177
178
179
180
                    members_dict[user.attributes['identities'][index]['extern_uid'].split(',')[0].replace('uid=', '')] = user.attributes['id']
                    identity_found = True

            if not identity_found:
                members_dict[user.attributes['username']] = user.attributes['id']
        else:
            members_dict[user.attributes['username']] = user.attributes['id']
181
182
183
184
185

    for member in members_dict:
        if member not in maints[project]:
            _member = proj.members.get(members_dict[member])
            if _member.attributes['access_level'] == 40:
186
187
                if not _member.attributes['username'].startswith(f"project_{ proj.attributes['id'] }_bot"):
                    proj.members.delete(members_dict[member])
188

189
                    print(f"Dropped master level access to { member } against repository { project } as maintainer entry is missing on the DOAP file")
190
191
            elif _member.attributes['access_level'] == 20:
                pass
192
            else:
Andrea Veri's avatar
Andrea Veri committed
193
194
                if not _member.attributes['username'].startswith(f"project_{ proj.attributes['id'] }_bot"):
                    proj.members.delete(members_dict[member])
195

Andrea Veri's avatar
Andrea Veri committed
196
                    print(f"Dropped level access { _member.attributes['access_level'] }, this means user { member } was added manually on project { project }, that is not necessary as permissions are inherited from the GNOME group by default")