From 9ccf8869782c040b879730f04bbf4f0f38e62598 Mon Sep 17 00:00:00 2001
From: Hans-Christoph Steiner <hans@eds.org>
Date: Mon, 19 Aug 2019 10:37:06 +0200
Subject: [PATCH] gitlab-ci: check active git repos to ensure they're available

This then edits the metadata file for apps with failing git repos, then
includes those edited files in the gitlab job artifacts.

fdroid/fdroiddata!5262
---
 .gitlab-ci.yml                       | 18 +++++++
 tools/check-git-repo-availability.py | 74 ++++++++++++++++++++++++++++
 2 files changed, 92 insertions(+)
 create mode 100755 tools/check-git-repo-availability.py

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 9e2407aca1..3935f51de9 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -63,6 +63,24 @@ pages:
     - gem install git_stats
     - git_stats generate --silent --out-path=public
 
+check_git_repos:
+  image: debian:buster-slim
+  stage: test
+  only:
+    - schedules
+  artifacts:
+    when: on_failure
+    expire_in: 1 month
+    paths:
+      - public
+  script:
+    - apt-get update
+    - apt-get -qy install --no-install-recommends ca-certificates git python3-yaml
+    - tools/check-git-repo-availability.py || export EXITVALUE=1
+    - test -d public || mkdir public
+    - cp `git status | grep -Eo 'metadata/.*\.yml'` public/ || true
+    - exit $EXITVALUE
+
 fdroid-buildserver:
   tags:
     - fdroid
diff --git a/tools/check-git-repo-availability.py b/tools/check-git-repo-availability.py
new file mode 100755
index 0000000000..bdb8b67767
--- /dev/null
+++ b/tools/check-git-repo-availability.py
@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+
+import glob
+import re
+import subprocess
+import sys
+import yaml
+
+if len(sys.argv) > 1:
+    files = sys.argv[1:]
+else:
+    files = sorted(glob.glob('metadata/*.yml'))
+
+errors = dict()
+for f in files:
+    if not f.endswith('.yml'):
+        print('\n' + f + ':\nThis only runs on YAML files (.yml), ignoring.')
+        continue
+    with open(f) as fp:
+        data = yaml.load(fp)
+
+    url = data.get('Repo')
+    if not url or 'NoSourceSince' in data.keys():
+        continue
+    if data['RepoType'] != 'git':
+        continue
+
+    # from class vcs_git() in fdroidserver/common.py
+    git_config = [
+        '-c', 'core.askpass=/bin/true',
+        '-c', 'core.sshCommand=/bin/false',
+        '-c', 'url.https://.insteadOf=ssh://',
+    ]
+    for domain in ('bitbucket.org', 'github.com', 'gitlab.com'):
+        git_config.append('-c')
+        git_config.append('url.https://u:p@' + domain + '/.insteadOf=git@' + domain + ':')
+        git_config.append('-c')
+        git_config.append('url.https://u:p@' + domain + '.insteadOf=git://' + domain)
+        git_config.append('-c')
+        git_config.append('url.https://u:p@' + domain + '.insteadOf=https://' + domain)
+    env = {
+        'GIT_TERMINAL_PROMPT': '0',
+        'GIT_ASKPASS': '/bin/true',
+        'SSH_ASKPASS': '/bin/true',
+        'GIT_SSH': '/bin/false',  # for git < 2.3
+    }
+
+    p = subprocess.run(['git', ] + git_config + ['ls-remote', '--exit-code', '-h', url],
+                       env=env,
+                       capture_output=True)
+    if p.returncode != 0:
+        with open(f) as fp:
+            raw = fp.read()
+        with open(f, 'w') as fp:
+            fp.write(re.sub(r'(Repo|RepoType):.*\n{1,2}', r'', raw))
+            builds = data.get('Builds')
+            if builds:
+                versionName = str(builds[-1]['versionName'])
+                # if YAML will think its a float, quote it
+                try:
+                    float(versionName)
+                    fp.write("\nNoSourceSince: '" + versionName + "'")
+                except ValueError:
+                    fp.write("\nNoSourceSince: " + versionName)
+                fp.write('\n')
+
+        print('\n' + f + ':')
+        print(p.stderr.decode())
+        errors[f] = p.stderr
+
+errorcount = len(errors)
+if errorcount > 0:
+    print('\nFound', errorcount, 'errors.')
+sys.exit(errorcount)