Merge branch 'master' into europeana

.github/CONTRIBUTING.md

@@ -4,6 +4,8 @@
 opened to market company APIs that offer paid solutions. This API list is not a marketing tool, but a tool to help the
 community build applications and use free, public APIs quickly and easily. Pull requests that are identified as marketing attempts will not be accepted.
 >
+> Please make sure the API you want to add has a full free access or at least a free tier before submitting.
+>
 > Thanks for understanding! :)
 
 ## Formatting
@@ -12,7 +14,7 @@ Current API entry format:
 
 | API | Description | Auth | HTTPS | CORS |
 | --- | --- | --- | --- | --- |
-| API Title(Link to API webpage) | Description of API | Does this API require authentication? * | Does the API support HTTPS? | Does the API support [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS)? * |
+| API Title(Link to API documentation) | Description of API | Does this API require authentication? * | Does the API support HTTPS? | Does the API support [CORS](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS)? * |
 
 Example entry:
 
@@ -33,6 +35,8 @@ Example entry:
 * `No` - _the API does not support CORS_
 * `Unknown` - _it is unknown if the API supports CORS_
 
+_Without proper [CORS configuration](https://developer.mozilla.org/en-US/docs/Web/HTTP/CORS) an API will only be usable server side._
+
 Please continue to follow the alphabetical ordering that is in place per section. Each table column should be padded with one space on either side.
 
 If an API seems to fall into multiple categories, please place the listing within the section most in line with the services offered through the API. For example, the Instagram API is listed under `Social` since it is mainly a social network, even though it could also apply to `Photography`.

.github/PULL_REQUEST_TEMPLATE.md

-Thank you for taking the time to work on a Pull Request for this project!
-
-To ensure your PR is dealt with swiftly please check the following:
-
-- [ ] Your submissions are formatted according to the guidelines in the [contributing guide](CONTRIBUTING.md)
-- [ ] Your additions are ordered alphabetically
-- [ ] Your submission has a useful description
+<!-- Thank you for taking the time to work on a Pull Request for this project! -->
+<!-- To ensure your PR is dealt with swiftly please check the following: -->
+- [ ] My submission is formatted according to the guidelines in the [contributing guide](CONTRIBUTING.md)
+- [ ] My addition is ordered alphabetically
+- [ ] My submission has a useful description
 - [ ] The description does not end with punctuation
-- [ ] Each table column should be padded with one space on either side
-- [ ] You have searched the repository for any relevant issues or pull requests
-- [ ] Any category you are creating has the minimum requirement of 3 items
+- [ ] Each table column is padded with one space on either side
+- [ ] I have searched the repository for any relevant issues or pull requests
+- [ ] Any category I am creating has the minimum requirement of 3 items
 - [ ] All changes have been [squashed][squash-link] into a single commit
 
 [squash-link]: <https://github.com/todotxt/todo.txt-android/wiki/Squash-All-Commits-Related-to-a-Single-Issue-into-a-Single-Commit>

.github/workflows/test.yml

+name: "Run tests"
+
+on:
+  schedule:
+    - cron:  '0 0 * * *'
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+env:
+  FORMAT_FILE: README.md
+
+jobs:
+  test:
+    name: 'Validate README.md'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Validate Markdown format
+        run: build/validate_format.py ${FORMAT_FILE}
+
+      - name: Validate pull request changes
+        run: build/github-pull.sh ${{ github.repository }} ${{ github.event.pull_request.number }} ${FORMAT_FILE}
+        if: github.event_name == 'pull_request'

.github/workflows/validate_links.yml

+name: "Validate links"
+
+on:
+  schedule:
+    - cron:  '0 0 * * *'
+  push:
+    branches:
+      - master
+
+env:
+  FORMAT_FILE: README.md
+
+jobs:
+  test:
+    name: 'Validate links'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v2
+
+      - name: Validate all links from README.md
+        run: build/validate_links.py ${FORMAT_FILE}

build/github-pull.sh

+#!/usr/bin/env bash
+
+set -e
+
+# Argument validation
+if [ $# -ne 3 ]; then
+    echo "Usage: $0 <github-repo> <pull-number> <format-file>"
+    exit 1
+fi
+
+# Assign variables
+GITHUB_REPOSITORY="$1"
+GITHUB_PULL_REQUEST="$2"
+FORMAT_FILE="$3"
+
+# Move to root of project
+cd "$GITHUB_WORKSPACE"
+
+# Determine files
+FORMAT_FILE="$( realpath "${FORMAT_FILE}" )"
+
+# Skip if build number could not be determined
+if [ -z "$GITHUB_REPOSITORY" -o -z "$GITHUB_PULL_REQUEST" ]; then
+    echo "No pull request and/or repository is provided"
+    exit 1
+fi
+
+# Pull changes on PR
+echo "running on Pull Request #$GITHUB_PULL_REQUEST"
+
+# Trick the URL validator python script into not seeing this as a URL
+DUMMY_SCHEME="https"
+DIFF_URL="$DUMMY_SCHEME://patch-diff.githubusercontent.com/raw/$GITHUB_REPOSITORY/pull/$GITHUB_PULL_REQUEST.diff"
+curl -L -o diff.txt "$DIFF_URL"
+
+# Construct diff
+echo "------- BEGIN DIFF -------"
+cat diff.txt
+echo "-------- END DIFF --------"
+cat diff.txt | egrep "\+" > additions.txt
+
+echo "------ BEGIN ADDITIONS -----"
+cat additions.txt
+echo "------- END ADDITIONS ------"
+LINK_FILE=additions.txt
+
+# Validate links
+echo "Running link validation..."
+./build/validate_links.py "$LINK_FILE"
+
+# Vebosity
+if [[ $? != 0 ]]; then
+    echo "link validation failed!"
+    exit 1
+else
+    echo "link validation passed!"
+fi

build/requirements.txt

 flake8>=3.5.0
-httplib2==0.9.2
+httplib2==0.18.0

build/validate_format.py

@@ -20,7 +20,6 @@ num_segments = 5
 
 errors = []
 title_links = []
-previous_links = []
 anchor_re = re.compile(anchor + '\s(.+)')
 section_title_re = re.compile('\*\s\[(.*)\]')
 link_re = re.compile('\[(.+)\]\((http.*)\)')
@@ -68,12 +67,6 @@ def check_entry(line_num, segments):
         title = title_re_match.group(1)
         if title.upper().endswith(' API'):
             add_error(line_num, 'Title should not end with "... API". Every entry is an API here!')
-        # do not allow duplicate links
-        link = title_re_match.group(2)
-        if link in previous_links:
-            add_error(line_num, 'Duplicate link - entries should only be included in one section')
-        else:
-            previous_links.append(link)
     # END Title
     # START Description
     # first character should be capitalized

build/validate_links.py

@@ -5,44 +5,92 @@ import re
 import socket
 import sys
 
+ignored_links = [
+    'https://github.com/public-apis/public-apis/actions?query=workflow%3A%22Run+tests%22', 
+    'https://github.com/public-apis/public-apis/workflows/Validate%20links/badge.svg?branch=master', 
+    'https://github.com/public-apis/public-apis/actions?query=workflow%3A%22Validate+links%22',
+    'https://github.com/davemachado/public-api',
+]
 
 def parse_links(filename):
     """Returns a list of URLs from text file"""
     with open(filename) as fp:
         data = fp.read()
     raw_links = re.findall(
-        'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+',
+        '((?:https?://|www\d{0,3}[.]|[a-z0-9.\-]+[.][a-z]{2,4}/)(?:[^\s()<>]+|\(([^\s()<>]+|(\([^\s()<>]+\)))*\))+(?:\(([^\s()<>]+|(\([^\s()<>]+\)))*\)|[^\s`!()\[\]{};:\'\".,<>?«»“”‘’]))',
         data)
-    links = [raw_link.replace(')', '') for raw_link in raw_links]
+    links = [raw_link[0] for raw_link in raw_links]
     return links
 
+def dup_links(links):
+    """Check for duplicated links"""
+    print(f'Checking for duplicated links...')
+    hasError = False
+    seen = {}
+    dupes = []
+
+    for link in links:
+        link = link.rstrip('/')
+        if link in ignored_links:
+            continue
+
+        if link not in seen:
+            seen[link] = 1
+        else:
+            if seen[link] == 1:
+                dupes.append(link)
+
+    if not dupes:
+        print(f"No duplicate links")
+    else:
+        print(f"Found duplicate links: {dupes}")  
+        hasError = True  
+    return hasError
 
 def validate_links(links):
     """Checks each entry in JSON file for live link"""
-    print('Validating {} links...'.format(len(links)))
-    errors = []
+    print(f'Validating {len(links)} links...')
+    hasError = False
     for link in links:
-        h = httplib2.Http(disable_ssl_certificate_validation=True, timeout=5)
+        h = httplib2.Http(disable_ssl_certificate_validation=True, timeout=25)
         try:
-            resp = h.request(link, 'HEAD')
+            resp = h.request(link, headers={
+                # Faking user agent as some hosting services block not-whitelisted UA
+                'user-agent': 'Mozilla/5.0'
+            })
             code = int(resp[0]['status'])
-            # check if status code is a client or server error
-            if code >= 404:
-                errors.append('{}: {}'.format(code, link))
+            # Checking status code errors
+            if (code >= 300):
+                hasError = True
+                print(f"ERR:CLT:{code} : {link}")
         except TimeoutError:
-            errors.append("TMO: " + link)
+            hasError = True
+            print(f"ERR:TMO: {link}")
         except socket.error as socketerror:
-            errors.append("SOC: {} : {}".format(socketerror, link))
-    return errors
-
+            hasError = True
+            print(f"ERR:SOC: {socketerror} : {link}")
+        except Exception as e:
+            hasError = True
+            # Ignore some exceptions which are not actually errors.
+            # The list below should be extended with other exceptions in the future if needed
+            if (-1 != str(e).find("[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed (_ssl.c:852)")):
+                print(f"ERR:SSL: {e} : {link}")
+            elif (-1 != str(e).find("Content purported to be compressed with gzip but failed to decompress.")):
+                print(f"ERR:GZP: {e} : {link}")
+            elif (-1 != str(e).find("Unable to find the server at")):
+                print(f"ERR:SRV: {e} : {link}")
+            else:
+                print(f"ERR:UKN: {e} : {link}")
+    return hasError
 
 if __name__ == "__main__":
     num_args = len(sys.argv)
     if num_args < 2:
         print("No .md file passed")
         sys.exit(1)
-    errors = validate_links(parse_links(sys.argv[1]))
-    if len(errors) > 0:
-        for err in errors:
-            print(err)
+    links = parse_links(sys.argv[1])
+    hasError = dup_links(links)
+    if not hasError:
+        hasError = validate_links(links)
+    if hasError:
         sys.exit(1)