Fix #2 and #3; improve debugging a bit

README.rst

@@ -38,7 +38,7 @@ were never merged.
        --gitlab-url http://workbench.dachary.org \
        --gitlab-token sxQJ67SQKihMrGWVf \
        --gitlab-repo ceph/ceph-backports \
-       --github-token 64933d355fda9844aadd4e224d \
+       --github-auth githubusername:64933d355fda9844aadd4e224d \
        --github-repo ceph/ceph \
        --ignore-closed
 
@@ -90,7 +90,7 @@ Hacking
     --gitlab-url http://workbench.dachary.org \
     --gitlab-token XXXXXXXXX \
     --gitlab-repo dachary/testrepo2 \
-    --github-token XXXXXXXXX \
+    --github-auth XXXXXXXXX \
     --github-repo dachary/testrepo \
     --ssh-public-key ~/.ssh/id_rsa.pub \
     --verbose  

github2gitlab/main.py

@@ -21,6 +21,7 @@
 import git
 import gitdb
 import hashlib
+from http.client import HTTPConnection
 import json
 import logging
 import os
@@ -33,6 +34,7 @@
 import shutil
 
 DESCRIPTION_MAX = 1024
+TITLE_MAX = 255
 
 logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s')
 
@@ -57,15 +59,18 @@ def __init__(self, args):
 
         if not self.args.gitlab_repo:
             self.args.gitlab_repo = self.args.github_repo
-        (self.args.gitlab_namespace,
-         self.args.gitlab_name) = self.args.gitlab_repo.split('/')
+
+        repo_parts = self.args.gitlab_repo.split('/')
+        self.args.gitlab_name = repo_parts.pop()
+        self.args.gitlab_namespace = '/'.join(repo_parts)
+
         self.args.gitlab_repo = parse.quote_plus(self.args.gitlab_repo)
 
         self.github = {
             'url': "https://api.github.com",
             'git': "https://github.com",
             'repo': self.args.github_repo,
-            'token': self.args.github_token,
+            'auth': self.args.github_auth or None
         }
         if self.args.branches:
             self.github['branches'] = self.args.branches.split(',')
@@ -74,18 +79,29 @@ def __init__(self, args):
             'host': self.args.gitlab_url,
             'name': self.args.gitlab_name,
             'namespace': self.args.gitlab_namespace,
+            'group_id': None,
             'url': self.args.gitlab_url + "/api/v4",
             'repo': self.args.gitlab_repo,
             'token': self.args.gitlab_token,
         }
 
         if self.args.verbose:
             level = logging.DEBUG
+            HTTPConnection.debuglevel = 1
         else:
             level = logging.INFO
 
+        logging.getLogger("urllib3").setLevel(level)
         logging.getLogger('github2gitlab').setLevel(level)
 
+        g = self.gitlab
+        url = g['url'] + "/groups"
+        query = {'private_token': g['token'], 'all_available': 'true', 'per_page': 10000}
+        groups = requests.get(url, params=query).json()
+        matches = list(filter(lambda group: group['full_path'] == g['namespace'].lower(), groups))
+        if any(matches):
+            g['group_id'] = matches[0]['id']
+
         self.tmpdir = "/tmp"
 
     @staticmethod
@@ -101,8 +117,8 @@ def get_parser():
                             required=True)
         parser.add_argument('--gitlab-repo',
                             help='Gitlab repo (for instance ceph/ceph)')
-        parser.add_argument('--github-token',
-                            help='GitHub authentication token')
+        parser.add_argument('--github-auth',
+                            help='GitHub auth credentials, in the form username:token')
         parser.add_argument('--github-repo',
                             help='GitHub repo (for instance ceph/ceph)',
                             required=True)
@@ -115,6 +131,9 @@ def get_parser():
         parser.add_argument('--ignore-closed', action='store_const',
                             const=True,
                             help='ignore pull requests closed and not merged')
+        parser.add_argument('--skip-add-key', action='store_const',
+                            const=True,
+                            help='do not attempt to add local SSH key to Gitlab')
         parser.add_argument('--skip-pull-requests', action='store_const',
                             const=True,
                             help='do not mirror PR to MR')
@@ -127,14 +146,18 @@ def get_parser():
         parser.add_argument('--clean', action='store_const',
                             const=True,
                             help='Remove the repo after sync')
+        parser.add_argument('--visibility',
+                            help='Visbility of created repos (public, internal, private)',
+                            default='public')
         return parser
 
     @staticmethod
     def factory(argv):
         return GitHub2GitLab(GitHub2GitLab.get_parser().parse_args(argv))
 
     def run(self):
-        self.add_key()
+        if not self.args.skip_add_key:
+            self.add_key()
         if self.add_project():
             self.unprotect_branches()
         self.git_mirror()
@@ -153,18 +176,20 @@ def sh(self, command):
             args=command,
             stdout=subprocess.PIPE,
             stderr=subprocess.STDOUT,
-            shell=True,
-            bufsize=1)
+            shell=True)
         lines = []
         with proc.stdout:
             for line in iter(proc.stdout.readline, b''):
                 line = line.decode('utf-8')
                 lines.append(line)
                 log.debug(str(line.strip()))
         if proc.wait() != 0:
+            output = "".join(lines)
+            log.error("Command failed: " + command + "\n" + output)
             raise subprocess.CalledProcessError(
                 returncode=proc.returncode,
-                cmd=command
+                cmd=command,
+                output=output
             )
         return "".join(lines)
 
@@ -179,9 +204,15 @@ def gitlab_create_remote(self, repo):
 
     def git_mirror(self):
         name = self.gitlab['name']
+        url = self.github['git']
+
+        if(self.github['auth']):
+            url = self.github['git'].replace('https://', 'https://{}@'.format(self.github['auth']))
+
         if not os.path.exists(name):
-            self.sh("git clone --bare " + self.github['git'] +
+            self.sh("git clone --bare " + url +
                     "/" + self.github['repo'] + " " + name)
+
         repo = git.Repo(name)
         os.chdir(name)
         if not hasattr(repo.remotes, 'gitlab'):
@@ -219,7 +250,7 @@ def git_mirror(self):
     def git_mirror_optimize(self, repo):
         self.sh("git fetch origin +refs/pull/*:refs/remotes/origin/pull/*")
         for head in repo.refs:
-            pr = re.search('^origin/pull/(\d+)/head$', head.name)
+            pr = re.search('^origin/pull/(\\d+)/head$', head.name)
             if not pr:
                 continue
             pr = pr.group(1)
@@ -285,15 +316,21 @@ def add_project(self):
         g = self.gitlab
         url = g['url'] + "/projects/" + g['repo']
         query = {'private_token': g['token']}
+        if g['group_id']:
+            query['group_id'] = g['group_id']
+
         if (requests.get(url, params=query).status_code == requests.codes.ok):
             log.debug("project " + url + " already exists")
             return None
         else:
-            log.info("add project " + g['repo'])
             url = g['url'] + "/projects"
-            query['public'] = 'true'
-            query['namespace'] = g['namespace']
+            query['visibility'] = self.args.visibility
             query['name'] = g['name']
+            if g['group_id']:
+                query['namespace_id'] = g['group_id']
+            else:
+                query['namespace'] = g['namespace']
+            log.info("add project " + g['repo'])
             result = requests.post(url, params=query)
             if result.status_code != requests.codes.created:
                 raise ValueError(result.text)
@@ -344,6 +381,13 @@ def field_equal(pull, pull_field, pull_value,
             merge_value = merge_value.replace(GitHub2GitLab.TAG_MERGED, '')
             return (pull_value[:DESCRIPTION_MAX] ==
                     merge_value[:DESCRIPTION_MAX])
+        elif pull_field == 'title':
+            if merge_value is None:
+                merge_value = ''
+            if pull_value is None:
+                pull_value = ''
+            return (pull_value[:TITLE_MAX] ==
+                    merge_value[:TITLE_MAX])
         else:
             return pull_value == merge_value
 
@@ -363,6 +407,8 @@ def field_update(pull, pull_field, pull_value,
             return ('state_event', value)
         elif pull_field == 'body':
             return (merge_field, pull_value[:DESCRIPTION_MAX])
+        elif pull_field == 'title':
+            return (merge_field, pull_value[:TITLE_MAX])
         else:
             return (merge_field, pull_value)
 
@@ -382,7 +428,7 @@ def sync(self):
                 target_branch = pull['base']['ref']
                 if (self.rev_parse(pull, source_branch) and
                         self.rev_parse(pull, target_branch)):
-                    data = {'title': pull['title'],
+                    data = {'title': pull['title'][:TITLE_MAX],
                             'source_branch': source_branch,
                             'target_branch': target_branch}
                     if pull['body']:
@@ -484,8 +530,8 @@ def get_pull_requests(self):
         "https://developer.github.com/v3/pulls/#list-pull-requests"
         g = self.github
         query = {'state': 'all'}
-        if self.args.github_token:
-            query['access_token'] = g['token']
+        if g['auth']:
+            query['access_token'] = g['auth'].split(':')[1]
 
         def f(pull):
             if self.args.ignore_closed:
@@ -511,7 +557,7 @@ def create_merge_request(self, query):
         g = self.gitlab
         query['private_token'] = g['token']
         url = g['url'] + "/projects/" + g['repo'] + "/merge_requests"
-        log.info('create_merge_request: ' + str(query))
+        log.debug('create_merge_request: ' + str(query))
         result = requests.post(url, params=query)
         if result.status_code != requests.codes.created:
             raise ValueError(result.text)
@@ -547,7 +593,7 @@ def put_merge_request(self, merge_request, updates):
         updates['private_token'] = g['token']
         url = (g['url'] + "/projects/" + g['repo'] + "/merge_requests/" +
                str(merge_request['iid']))
-        log.info('update_merge_request: ' + url + ' <= ' + str(updates))
+        log.debug('update_merge_request: ' + url + ' <= ' + str(updates))
         return requests.put(url, params=updates).json()
 
     def verify_merge_update(self, updates, result):