gitlab-ci: check active git repos to ensure they're available

This then edits the metadata file for apps with failing git repos, then
includes those edited files in the gitlab job artifacts.

fdroid/fdroiddata!5262

.gitlab-ci.yml

@@ -63,6 +63,24 @@ pages:
     - gem install git_stats
     - git_stats generate --silent --out-path=public
 
+check_git_repos:
+  image: debian:buster-slim
+  stage: test
+  only:
+    - schedules
+  artifacts:
+    when: on_failure
+    expire_in: 1 month
+    paths:
+      - public
+  script:
+    - apt-get update
+    - apt-get -qy install --no-install-recommends ca-certificates git python3-yaml
+    - tools/check-git-repo-availability.py || export EXITVALUE=1
+    - test -d public || mkdir public
+    - cp `git status | grep -Eo 'metadata/.*\.yml'` public/ || true
+    - exit $EXITVALUE
+
 fdroid-buildserver:
   tags:
     - fdroid

tools/check-git-repo-availability.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+import glob
+import re
+import subprocess
+import sys
+import yaml
+
+if len(sys.argv) > 1:
+    files = sys.argv[1:]
+else:
+    files = sorted(glob.glob('metadata/*.yml'))
+
+errors = dict()
+for f in files:
+    if not f.endswith('.yml'):
+        print('\n' + f + ':\nThis only runs on YAML files (.yml), ignoring.')
+        continue
+    with open(f) as fp:
+        data = yaml.load(fp)
+
+    url = data.get('Repo')
+    if not url or 'NoSourceSince' in data.keys():
+        continue
+    if data['RepoType'] != 'git':
+        continue
+
+    # from class vcs_git() in fdroidserver/common.py
+    git_config = [
+        '-c', 'core.askpass=/bin/true',
+        '-c', 'core.sshCommand=/bin/false',
+        '-c', 'url.https://.insteadOf=ssh://',
+    ]
+    for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
+        git_config.append('-c')
+        git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
+        git_config.append('-c')
+        git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
+        git_config.append('-c')
+        git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
+    env = {
+        'GIT_TERMINAL_PROMPT': '0',
+        'GIT_ASKPASS': '/bin/true',
+        'SSH_ASKPASS': '/bin/true',
+        'GIT_SSH': '/bin/false',  # for git < 2.3
+    }
+
+    p = subprocess.run(['git', ] + git_config + ['ls-remote', '--exit-code', '-h', url],
+                       env=env,
+                       capture_output=True)
+    if p.returncode != 0:
+        with open(f) as fp:
+            raw = fp.read()
+        with open(f, 'w') as fp:
+            fp.write(re.sub(r'(Repo|RepoType):.*\n{1,2}', r'', raw))
+            builds = data.get('Builds')
+            if builds:
+                versionName = str(builds[-1]['versionName'])
+                # if YAML will think its a float, quote it
+                try:
+                    float(versionName)
+                    fp.write("\nNoSourceSince: '" + versionName + "'")
+                except ValueError:
+                    fp.write("\nNoSourceSince: " + versionName)
+                fp.write('\n')
+
+        print('\n' + f + ':')
+        print(p.stderr.decode())
+        errors[f] = p.stderr
+
+errorcount = len(errors)
+if errorcount > 0:
+    print('\nFound', errorcount, 'errors.')
+sys.exit(errorcount)