Merge branch 'improve-gitlab-ci' into 'master'

Improve gitlab-ci tests

See merge request fdroid/fdroiddata!2828

.gitlab-ci.yml

@@ -8,13 +8,25 @@ before_script:
 
 lint:
   script:
+    # if this is a merge request fork, then only check relevant apps
+    - if [ "$CI_PROJECT_NAMESPACE" != "fdroid" ]; then
+        git fetch https://gitlab.com/fdroid/fdroiddata;
+        test -d build || mkdir build;
+        for f in `git diff --name-only FETCH_HEAD`; do
+           appid=`echo $f | sed -n -e 's,^metadata/\(.*\)\.txt,\1,p'
+                                   -e 's,^metadata/\(.*\)\.yml,\1,p'`;
+           export CHANGED="$CHANGED $appid";
+           grep -q "^Repo *Type\W *git" $f && git -C build clone `sed -n "s,^Repo *:,,p" $f` $appid;
+        done;
+        ./tools/audit-gradle.py $CHANGED;
+      fi
     - export EXITVALUE=0
-    - fdroid lint -f || {
+    - fdroid lint -f $CHANGED || {
           export EXITVALUE=1;
           printf "\nThese files have lint issues:\n";
-          fdroid rewritemeta -l;
+          fdroid rewritemeta -l $CHANGED;
           printf "\nThese are the formatting issues:\n";
-          fdroid rewritemeta;
+          fdroid rewritemeta $CHANGED;
           git --no-pager diff --color=always;
       }
     - exit $EXITVALUE

tools/audit-gradle.py

@@ -0,0 +1,30 @@
+#!/usr/bin/env python3
+
+
+import os
+import re
+import sys
+
+# find all repositories that use plain HTTP urls (e.g. not HTTPS)
+url_pattern = re.compile('repositories\s*{[^}]*http://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+[^}]*}', re.DOTALL)
+
+exit_value = 0
+for appid in sys.argv:
+    gitdir = os.path.join('build', appid)
+    if not os.path.isdir(gitdir):
+        continue
+    for root, dirs, files in os.walk(gitdir):
+        for f in files:
+            if f.endswith('.gradle'):
+                path = os.path.join(root, f)
+                with open(path) as fp:
+                    data = fp.read()
+                for url in url_pattern.findall(data):
+                    print('Found plain HTTP URL for gradle repository:\n%s\n%s'
+                          % (path, url))
+                    exit_value += 1
+
+if exit_value:
+    print('gradle build uses plain HTTP URLs for repositories!  This is insecure!')
+    print('https://max.computer/blog/how-to-take-over-the-computer-of-any-java-or-clojure-or-scala-developer/')
+sys.exit(exit_value)